diff --git a/.github/actions/cache-restore/action.yml b/.github/actions/cache-restore/action.yml index 9919ad6aea30f8..286f615805e6b7 100644 --- a/.github/actions/cache-restore/action.yml +++ b/.github/actions/cache-restore/action.yml @@ -95,6 +95,15 @@ runs: echo "toolchain_version=$(echo ${version} | cut -d'.' -f1)" >> $GITHUB_OUTPUT fi + - name: 'Show cache storage' + shell: bash + env: + GH_TOKEN: ${{ github.token }} + run: | + echo "::group::gh cache list" + gh cache list --limit 100 + echo "::endgroup::" + - name: 'Toolchain Prebuilt Cache' uses: actions/cache/restore@v4 id: 'toolchain-prebuilt' @@ -154,22 +163,16 @@ runs: uses: actions/cache@v4 with: path: ${{ inputs.download_cache_path }}/TZDB - key: TimeZoneData-${{ hashFiles('Meta/CMake/time_zone_data.cmake') }}-${{ steps.date-stamp.outputs.timestamp }} - restore-keys: | - TimeZoneData-${{ hashFiles('Meta/CMake/time_zone_data.cmake') }} + key: TimeZoneData-${{ hashFiles('Meta/CMake/time_zone_data.cmake') }} - name: 'UnicodeData cache' uses: actions/cache@v4 with: path: ${{ inputs.download_cache_path }}/UCD - key: UnicodeData-${{ hashFiles('Meta/CMake/unicode_data.cmake') }}-${{ steps.date-stamp.outputs.timestamp }} - restore-keys: | - UnicodeData-${{ hashFiles('Meta/CMake/unicode_data.cmake') }} + key: UnicodeData-${{ hashFiles('Meta/CMake/unicode_data.cmake') }} - name: 'UnicodeLocale cache' uses: actions/cache@v4 with: path: ${{ inputs.download_cache_path }}/CLDR - key: UnicodeLocale-${{ hashFiles('Meta/CMake/locale_data.cmake') }}-${{ steps.date-stamp.outputs.timestamp }} - restore-keys: | - UnicodeLocale-${{ hashFiles('Meta/CMake/locale_data.cmake') }} + key: UnicodeLocale-${{ hashFiles('Meta/CMake/locale_data.cmake') }} diff --git a/.github/actions/cache-save/action.yml b/.github/actions/cache-save/action.yml index 5c8c5912375460..1e0ba9d019ddaf 100644 --- a/.github/actions/cache-save/action.yml +++ b/.github/actions/cache-save/action.yml @@ -51,6 +51,16 @@ inputs: runs: using: "composite" steps: + - name: 'Show cache storage' + shell: bash + env: + GH_TOKEN: + ${{ github.token }} + run: | + echo "::group::gh cache list" + gh cache list --limit 100 + echo "::endgroup::" + - name: 'Toolchain Prebuilt Cache' uses: actions/cache/save@v4 # Do not waste time and storage space by updating the toolchain cache from a PR, @@ -82,7 +92,7 @@ runs: - name: 'Serenity Compiler Cache' uses: actions/cache/save@v4 - if: ${{ inputs.serenity_ccache_path != '' }} + if: ${{ github.event_name != 'pull_request' && inputs.serenity_ccache_path != '' }} with: path: ${{ inputs.serenity_ccache_path }} key: ${{ inputs.serenity_ccache_primary_key }} @@ -95,3 +105,12 @@ runs: echo "Serenity Compiler Cache" CCACHE_DIR=${{ inputs.serenity_ccache_path }} ccache -s + + - name: 'Show cache storage' + shell: bash + env: + GH_TOKEN: ${{ github.token }} + run: | + echo "::group::gh cache list" + gh cache list --limit 100 + echo "::endgroup::" diff --git a/.github/actions/setup/action.yml b/.github/actions/setup/action.yml index b383ed50a4a920..c5c8c8bc600a36 100644 --- a/.github/actions/setup/action.yml +++ b/.github/actions/setup/action.yml @@ -69,7 +69,7 @@ runs: sudo apt-get update sudo apt-get remove clang-{14..18} clang++-{14..18} libclang-{14..18}-dev llvm-{14..18} - sudo apt-get install clang-format-20 ccache e2fsprogs libmpfr-dev libmpc-dev optipng qemu-utils qemu-system-i386 generate-ninja libegl1-mesa-dev + sudo apt-get install clang-format-20 ccache e2fsprogs libmpfr-dev libmpc-dev optipng qemu-utils qemu-system-i386 qemu-system-arm qemu-system-riscv64 generate-ninja libegl1-mesa-dev sudo apt-get install clang-20 clang++-20 llvm-20 llvm-20-dev - name: Enable KVM group perms diff --git a/.github/workflows/dev-container.yml b/.github/workflows/dev-container.yml index 587fcd02adda0c..294427631c5f44 100644 --- a/.github/workflows/dev-container.yml +++ b/.github/workflows/dev-container.yml @@ -22,7 +22,7 @@ jobs: steps: - name: Checkout - uses: actions/checkout@v5 + uses: actions/checkout@v6.0.1 - name: Login to GitHub Container Registry uses: docker/login-action@v3 diff --git a/.github/workflows/label-pull-requests.yml b/.github/workflows/label-pull-requests.yml index d7355d98eee6d7..3faa9a33ab4461 100644 --- a/.github/workflows/label-pull-requests.yml +++ b/.github/workflows/label-pull-requests.yml @@ -13,9 +13,9 @@ jobs: label_pull_request: runs-on: ubuntu-24.04 if: always() && github.repository == 'SerenityOS/serenity' - + steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@v6.0.1 - name: Label pull request uses: actions/github-script@v8 diff --git a/.github/workflows/lagom-template.yml b/.github/workflows/lagom-template.yml index faa3ab4f44fe46..673310107ca61d 100644 --- a/.github/workflows/lagom-template.yml +++ b/.github/workflows/lagom-template.yml @@ -31,10 +31,10 @@ jobs: # Pull requests can trail behind `master` and can cause breakage if merging before running the CI checks on an updated branch. # Luckily, GitHub creates and maintains a merge branch that is updated whenever the target or source branch is modified. By # checking this branch out, we gain a stabler `master` at the cost of reproducibility. - - uses: actions/checkout@v5 + - uses: actions/checkout@v6.0.1 if: ${{ github.event_name != 'pull_request' }} - - uses: actions/checkout@v5 + - uses: actions/checkout@v6.0.1 if: ${{ github.event_name == 'pull_request' }} with: ref: refs/pull/${{ github.event.pull_request.number }}/merge diff --git a/.github/workflows/manpages.yml b/.github/workflows/manpages.yml index 8c1eb881f54450..a652dda71ca54d 100644 --- a/.github/workflows/manpages.yml +++ b/.github/workflows/manpages.yml @@ -9,16 +9,21 @@ on: jobs: convert_using_pandoc: runs-on: ubuntu-24.04 - if: always() && github.repository == 'SerenityOS/serenity' && github.ref == 'refs/heads/master' + if: github.repository == 'SerenityOS/serenity' && github.ref == 'refs/heads/master' steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@v6.0.1 + with: + persist-credentials: false + - uses: r-lib/actions/setup-pandoc@v2 with: pandoc-version: '2.13' + - name: Actually build website run: ./Meta/build-manpages-website.sh + - name: Deploy to GitHub pages - uses: JamesIves/github-pages-deploy-action@v4.7.4 + uses: JamesIves/github-pages-deploy-action@v4.7.6 with: git-config-name: BuggieBot git-config-email: buggiebot@serenityos.org diff --git a/.github/workflows/pull-requests-label-workflows.yml b/.github/workflows/pull-requests-label-workflows.yml new file mode 100644 index 00000000000000..067d85b60a37f0 --- /dev/null +++ b/.github/workflows/pull-requests-label-workflows.yml @@ -0,0 +1,15 @@ +name: Label Workflows + +on: + pull_request: + types: [ labeled, opened, synchronize, reopened ] + +jobs: + Coverage: + if: github.repository == 'SerenityOS/serenity' && contains(github.event.pull_request.labels.*.name, '📑 pr-run-coverage-build') + uses: ./.github/workflows/serenity-template.yml + with: + toolchain: 'Clang' + os: ubuntu-24.04 + arch: 'x86_64' + coverage: 'ON' diff --git a/.github/workflows/serenity-template.yml b/.github/workflows/serenity-template.yml index c54e528e4db52d..7f286d1fedfc1e 100644 --- a/.github/workflows/serenity-template.yml +++ b/.github/workflows/serenity-template.yml @@ -36,10 +36,10 @@ jobs: # Pull requests can trail behind `master` and can cause breakage if merging before running the CI checks on an updated branch. # Luckily, GitHub creates and maintains a merge branch that is updated whenever the target or source branch is modified. By # checking this branch out, we gain a stabler `master` at the cost of reproducibility. - - uses: actions/checkout@v5 + - uses: actions/checkout@v6.0.1 if: ${{ github.event_name != 'pull_request' }} - - uses: actions/checkout@v5 + - uses: actions/checkout@v6.0.1 if: ${{ github.event_name == 'pull_request' }} with: ref: refs/pull/${{ github.event.pull_request.number }}/merge @@ -50,6 +50,11 @@ jobs: os: 'Serenity' arch: ${{ inputs.arch }} + - name: Set Disk Size for Coverage Builds + if: ${{ inputs.coverage == 'ON' }} + # Using 5GiB here is arbitrary. + run: echo "SERENITY_DISK_SIZE_BYTES=5368709120" >> $GITHUB_ENV + # === PREPARE FOR BUILDING === - name: Lint @@ -135,6 +140,7 @@ jobs: CCACHE_DIR: ${{ env.SERENITY_CCACHE_DIR }} - name: Save Caches + if: ${{ inputs.coverage == 'OFF' }} uses: ./.github/actions/cache-save with: arch: ${{ inputs.arch }} @@ -157,7 +163,7 @@ jobs: run: ninja install && ninja qemu-image - name: Run On-Target Tests - if: ${{ inputs.debug_options == 'NORMAL_DEBUG' && inputs.arch != 'aarch64' && inputs.arch != 'riscv64' }} + if: ${{ inputs.debug_options == 'NORMAL_DEBUG' }} working-directory: ${{ steps.build-parameters.outputs.build_directory }} env: SERENITY_QEMU_CPU: "max,vmx=off" @@ -200,11 +206,10 @@ jobs: SERENITY_ARCH: ${{ inputs.arch }} # FIXME: Deploy the static html pages somewhere - # FIXME: Alter script to also (instead?) produce a raw coverage.txt file for ingestion into sonar cloud # Note: tmp_profile_data/Coverage.profdata has the entire combined profile data, but creating the raw txt requires # all of the instrumented binaries and the profdata file. - name: Upload Coverage Results - if: ${{ inputs.coverage == 'ON' }} + if: ${{ inputs.coverage == 'ON' && github.event_name != 'pull_request' }} uses: actions/upload-artifact@v5 with: name: serenity-coverage diff --git a/.github/workflows/social-media.yml b/.github/workflows/social-media.yml index 7c578354ef3e67..5cc3daa5a185cf 100644 --- a/.github/workflows/social-media.yml +++ b/.github/workflows/social-media.yml @@ -3,32 +3,12 @@ name: Social media notifications on: [ push ] jobs: - notify_twitter: - runs-on: ubuntu-24.04 - if: always() && github.repository == 'SerenityOS/serenity' && github.ref == 'refs/heads/master' - - steps: - - uses: actions/checkout@v5 - - uses: actions/setup-node@v6 - with: - node-version: '14' - - run: npm i twit - - run: | - node ${{ github.workspace }}/Meta/tweet-commits.js << 'EOF' - ${{ toJSON(github.event) }} - EOF - env: - CONSUMER_KEY: ${{ secrets.CONSUMER_KEY }} - CONSUMER_SECRET: ${{ secrets.CONSUMER_SECRET }} - ACCESS_TOKEN: ${{ secrets.ACCESS_TOKEN }} - ACCESS_TOKEN_SECRET: ${{ secrets.ACCESS_TOKEN_SECRET }} - notify_mastodon: runs-on: ubuntu-24.04 if: always() && github.repository == 'SerenityOS/serenity' && github.ref == 'refs/heads/master' steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@v6.0.1 - uses: actions/setup-node@v6 with: node-version: '14' diff --git a/.github/workflows/wasm.yml b/.github/workflows/wasm.yml index 998c2aee29fe97..ad63c7663b4907 100644 --- a/.github/workflows/wasm.yml +++ b/.github/workflows/wasm.yml @@ -13,16 +13,19 @@ jobs: if: github.repository == 'SerenityOS/serenity' strategy: fail-fast: false + steps: - name: Checkout SerenityOS/serenity - uses: actions/checkout@v5 + uses: actions/checkout@v6.0.1 + with: + persist-credentials: false - name: Checkout SerenityOS/libjs-data libjs-wasm - uses: actions/checkout@v5 + uses: actions/checkout@v6.0.1 with: - repository: SerenityOS/libjs-data path: libjs-data ref: libjs-wasm + repository: SerenityOS/libjs-data - name: "Set up environment" uses: ./.github/actions/setup @@ -97,7 +100,7 @@ jobs: tar --exclude='.[^/]*' -czvf libjs-wasm.tar.gz -C ${{ github.workspace }}/libjs-data . - name: Deploy to GitHub - uses: JamesIves/github-pages-deploy-action@v4.7.4 + uses: JamesIves/github-pages-deploy-action@v4.7.6 if: github.ref == 'refs/heads/master' with: git-config-name: BuggieBot @@ -108,7 +111,6 @@ jobs: folder: ${{ github.workspace }}/libjs-data - name: Upload artifact package - if: github.ref == 'refs/heads/master' uses: actions/upload-artifact@v5 with: name: serenity-js-wasm diff --git a/AK/ByteString.h b/AK/ByteString.h index fc859ce17bf03a..fd3974cae14287 100644 --- a/AK/ByteString.h +++ b/AK/ByteString.h @@ -6,6 +6,10 @@ #pragma once +#ifdef KERNEL +# error This file should not be included in the kernel. Use KString instead. +#endif + #include #include #include diff --git a/AK/CMakeLists.txt b/AK/CMakeLists.txt index 03237b41092d15..66c75c28a1b2d0 100644 --- a/AK/CMakeLists.txt +++ b/AK/CMakeLists.txt @@ -15,6 +15,7 @@ set(AK_SOURCES FuzzyMatch.cpp GenericLexer.cpp Hex.cpp + InternetChecksum.cpp JsonObject.cpp JsonParser.cpp JsonPath.cpp diff --git a/AK/Enumerate.h b/AK/Enumerate.h index d824605e7dd95e..0203f9b3dcfc1c 100644 --- a/AK/Enumerate.h +++ b/AK/Enumerate.h @@ -11,8 +11,9 @@ namespace AK { namespace Detail { + template -class Enumerator { +struct Enumerator { using IteratorType = decltype(declval().begin()); using ValueType = decltype(*declval()); @@ -21,35 +22,28 @@ class Enumerator { ValueType value; }; -public: - Enumerator(Iterable&& iterable) - : m_iterable(forward(iterable)) - , m_iterator(m_iterable.begin()) - , m_end(m_iterable.end()) - { - } - - Enumerator const& begin() const { return *this; } - Enumerator const& end() const { return *this; } + struct Iterator { + Enumeration operator*() { return { index, *iterator }; } + Enumeration operator*() const { return { index, *iterator }; } - Enumeration operator*() { return { m_index, *m_iterator }; } - Enumeration operator*() const { return { m_index, *m_iterator }; } + bool operator!=(Iterator const& other) const { return iterator != other.iterator; } - bool operator!=(Enumerator const&) const { return m_iterator != m_end; } + void operator++() + { + ++index; + ++iterator; + } - void operator++() - { - ++m_index; - ++m_iterator; - } + size_t index { 0 }; + IteratorType iterator; + }; -private: - Iterable m_iterable; + Iterator begin() { return { 0, iterable.begin() }; } + Iterator end() { return { 0, iterable.end() }; } - size_t m_index { 0 }; - IteratorType m_iterator; - IteratorType const m_end; + Iterable iterable; }; + } template diff --git a/AK/Format.cpp b/AK/Format.cpp index 17c542a7c31bd4..8fc79b3cd9c896 100644 --- a/AK/Format.cpp +++ b/AK/Format.cpp @@ -9,7 +9,6 @@ #include #include #include -#include #include #include @@ -24,16 +23,14 @@ # include #else # include +# include +# include # include # include # include # include #endif -#ifndef KERNEL -# include -#endif - namespace AK { namespace { diff --git a/AK/GenericLexer.h b/AK/GenericLexer.h index fc7990c6c90b5a..99a34ba1b713b5 100644 --- a/AK/GenericLexer.h +++ b/AK/GenericLexer.h @@ -9,9 +9,12 @@ #include #include #include -#include #include +#ifndef KERNEL +# include +#endif + namespace AK { class GenericLexer { diff --git a/AK/InternetChecksum.cpp b/AK/InternetChecksum.cpp new file mode 100644 index 00000000000000..0ab0413804c6c6 --- /dev/null +++ b/AK/InternetChecksum.cpp @@ -0,0 +1,41 @@ +/* + * Copyright (c) 2025, the SerenityOS developers. + * + * SPDX-License-Identifier: BSD-2-Clause + */ + +#include +#include + +namespace AK { + +InternetChecksum::InternetChecksum(ReadonlyBytes bytes) +{ + update(bytes); +} + +void InternetChecksum::update(ReadonlyBytes bytes) +{ + VERIFY(!m_uneven_payload); + for (size_t i = 0; i < bytes.size() / sizeof(u16); ++i) + m_state += ByteReader::load16(bytes.offset_pointer(i * sizeof(u16))); + if (bytes.size() % 2 == 1) { + m_state += bytes.last(); + m_uneven_payload = true; + } +} + +NetworkOrdered InternetChecksum::digest() +{ + u32 output = m_state; + // While there are any bits above the bottom 16... + while (output >> 16) + // Drop the top bits, and add the carries to the sum. + output = (output & 0xFFFF) + (output >> 16); + + // Return the one's complement (this is already network-ordered, so we + // bypass the constructor of that). + return bit_cast>(static_cast(~output)); +} + +} diff --git a/AK/InternetChecksum.h b/AK/InternetChecksum.h new file mode 100644 index 00000000000000..562e3e2fad7936 --- /dev/null +++ b/AK/InternetChecksum.h @@ -0,0 +1,32 @@ +/* + * Copyright (c) 2025, the SerenityOS developers. + * + * SPDX-License-Identifier: BSD-2-Clause + */ + +#pragma once + +#include +#include +#include + +namespace AK { + +class InternetChecksum { +public: + InternetChecksum() = default; + InternetChecksum(ReadonlyBytes); + + void update(ReadonlyBytes); + NetworkOrdered digest(); + +private: + u32 m_state { 0 }; + bool m_uneven_payload { false }; +}; + +} + +#ifdef USING_AK_GLOBALLY +using AK::InternetChecksum; +#endif diff --git a/AK/MemMem.h b/AK/MemMem.h index 3853858d4878b1..b109c8ac3d89eb 100644 --- a/AK/MemMem.h +++ b/AK/MemMem.h @@ -48,7 +48,7 @@ requires(requires { (*haystack_begin).data(); (*haystack_begin).size(); }) { auto prepare_kmp_partial_table = [&] { Vector table; - table.resize(needle.size()); + table.try_resize(needle.size()).release_value_but_fixme_should_propagate_errors(); size_t position = 1; int candidate = 0; diff --git a/AK/Stack.h b/AK/Stack.h index acdc3830fa97af..a1c618091e57a4 100644 --- a/AK/Stack.h +++ b/AK/Stack.h @@ -49,7 +49,7 @@ class Stack { if (is_empty()) return false; - m_stack.resize_and_keep_capacity(m_stack.size() - 1); + m_stack.shrink(m_stack.size() - 1, true); return true; } diff --git a/AK/StdLibExtraDetails.h b/AK/StdLibExtraDetails.h index 273de31d09bace..88d95410c7f494 100644 --- a/AK/StdLibExtraDetails.h +++ b/AK/StdLibExtraDetails.h @@ -526,6 +526,9 @@ inline constexpr bool IsMoveAssignable = IsAssignable, Add template inline constexpr bool IsTriviallyMoveAssignable = IsTriviallyAssignable, AddRvalueReference>; +template +inline constexpr bool IsPolymorphic = __is_polymorphic(T); + template typename U> inline constexpr bool IsSpecializationOf = false; @@ -666,6 +669,7 @@ using AK::Detail::IsOneOf; using AK::Detail::IsOneOfIgnoringCV; using AK::Detail::IsPOD; using AK::Detail::IsPointer; +using AK::Detail::IsPolymorphic; using AK::Detail::IsRvalueReference; using AK::Detail::IsSame; using AK::Detail::IsSameIgnoringCV; diff --git a/AK/String.h b/AK/String.h index 3ca2ace489a2e2..0d53956db4dd29 100644 --- a/AK/String.h +++ b/AK/String.h @@ -7,6 +7,10 @@ #pragma once +#ifdef KERNEL +# error This file should not be included in the kernel. Use KString instead. +#endif + #include #include #include diff --git a/AK/StringBuilder.cpp b/AK/StringBuilder.cpp index 57a2c118d6e86d..8a02647657fac2 100644 --- a/AK/StringBuilder.cpp +++ b/AK/StringBuilder.cpp @@ -8,7 +8,6 @@ #include #include #include -#include #include #include #include @@ -17,6 +16,7 @@ #ifndef KERNEL # include # include +# include # include #endif diff --git a/AK/StringUtils.cpp b/AK/StringUtils.cpp index cfddedc817d17c..474ea29fef2691 100644 --- a/AK/StringUtils.cpp +++ b/AK/StringUtils.cpp @@ -9,7 +9,6 @@ #include #include #include -#include #include #include #include @@ -22,6 +21,7 @@ #else # include # include +# include # include #endif @@ -29,6 +29,7 @@ namespace AK { namespace StringUtils { +#ifndef KERNEL bool matches(StringView str, StringView mask, CaseSensitivity case_sensitivity, Vector* match_spans) { auto record_span = [&match_spans](size_t start, size_t length) { @@ -93,6 +94,7 @@ bool matches(StringView str, StringView mask, CaseSensitivity case_sensitivity, return string_ptr == string_end && mask_ptr == mask_end; } +#endif template Optional convert_to_int(StringView str, TrimWhitespace trim_whitespace) @@ -431,6 +433,7 @@ Optional find_last_not(StringView haystack, char needle) return {}; } +#ifndef KERNEL Vector find_all(StringView haystack, StringView needle) { Vector positions; @@ -446,6 +449,7 @@ Vector find_all(StringView haystack, StringView needle) } return positions; } +#endif Optional find_any_of(StringView haystack, StringView needles, SearchDirection direction) { diff --git a/AK/StringUtils.h b/AK/StringUtils.h index 1f29fdff748bca..323edd12b25e42 100644 --- a/AK/StringUtils.h +++ b/AK/StringUtils.h @@ -73,7 +73,10 @@ struct MaskSpan { namespace StringUtils { +#ifndef KERNEL bool matches(StringView str, StringView mask, CaseSensitivity = CaseSensitivity::CaseInsensitive, Vector* match_spans = nullptr); +#endif + template Optional convert_to_int(StringView, TrimWhitespace = TrimWhitespace::Yes); template @@ -99,7 +102,11 @@ Optional find(StringView haystack, StringView needle, size_t start = 0); Optional find_last(StringView haystack, char needle); Optional find_last(StringView haystack, StringView needle); Optional find_last_not(StringView haystack, char needle); + +#ifndef KERNEL Vector find_all(StringView haystack, StringView needle); +#endif + enum class SearchDirection { Forward, Backward diff --git a/AK/StringView.cpp b/AK/StringView.cpp index 7d23e1aa3a6d45..a7d828a3af39f5 100644 --- a/AK/StringView.cpp +++ b/AK/StringView.cpp @@ -63,7 +63,7 @@ Vector StringView::split_view(StringView separator, SplitBehavior sp { Vector parts; for_each_split_view(separator, split_behavior, [&](StringView view) { - parts.append(view); + parts.try_append(view).release_value_but_fixme_should_propagate_errors(); }); return parts; } @@ -110,6 +110,7 @@ static void for_each_line(StringView string, Callback&& callback) callback(string.substring_view(substart, taillen)); } +#ifndef KERNEL Vector StringView::lines(ConsiderCarriageReturn consider_carriage_return) const { if (is_empty()) @@ -123,6 +124,7 @@ Vector StringView::lines(ConsiderCarriageReturn consider_carriage_re return lines; } +#endif size_t StringView::count_lines(ConsiderCarriageReturn consider_carriage_return) const { @@ -162,6 +164,7 @@ bool StringView::ends_with(StringView str, CaseSensitivity case_sensitivity) con return StringUtils::ends_with(*this, str, case_sensitivity); } +#ifndef KERNEL bool StringView::matches(StringView mask, Vector& mask_spans, CaseSensitivity case_sensitivity) const { return StringUtils::matches(*this, mask, case_sensitivity, &mask_spans); @@ -171,6 +174,7 @@ bool StringView::matches(StringView mask, CaseSensitivity case_sensitivity) cons { return StringUtils::matches(*this, mask, case_sensitivity); } +#endif bool StringView::contains(char needle) const { @@ -267,7 +271,6 @@ ByteString StringView::replace(StringView needle, StringView replacement, Replac { return StringUtils::replace(*this, needle, replacement, replace_mode); } -#endif Vector StringView::find_all(StringView needle) const { @@ -297,5 +300,6 @@ Vector StringView::split_view_if(Function const& predica v.append(substring_view(substart, taillen)); return v; } +#endif } diff --git a/AK/StringView.h b/AK/StringView.h index f6c1d2fc3c9f5e..8cfdd243887d6a 100644 --- a/AK/StringView.h +++ b/AK/StringView.h @@ -100,8 +100,12 @@ class StringView { [[nodiscard]] bool ends_with(StringView, CaseSensitivity = CaseSensitivity::CaseSensitive) const; [[nodiscard]] bool starts_with(char) const; [[nodiscard]] bool ends_with(char) const; + +#ifndef KERNEL [[nodiscard]] bool matches(StringView mask, CaseSensitivity = CaseSensitivity::CaseInsensitive) const; [[nodiscard]] bool matches(StringView mask, Vector&, CaseSensitivity = CaseSensitivity::CaseInsensitive) const; +#endif + [[nodiscard]] bool contains(char) const; [[nodiscard]] bool contains(u32) const; [[nodiscard]] bool contains(StringView, CaseSensitivity = CaseSensitivity::CaseSensitive) const; @@ -125,7 +129,9 @@ class StringView { [[nodiscard]] Optional find_last(StringView needle) const { return StringUtils::find_last(*this, needle); } [[nodiscard]] Optional find_last_not(char needle) const { return StringUtils::find_last_not(*this, needle); } +#ifndef KERNEL [[nodiscard]] Vector find_all(StringView needle) const; +#endif using SearchDirection = StringUtils::SearchDirection; [[nodiscard]] Optional find_any_of(StringView needles, SearchDirection direction = SearchDirection::Forward) const { return StringUtils::find_any_of(*this, needles, direction); } @@ -147,7 +153,9 @@ class StringView { [[nodiscard]] Vector split_view(char, SplitBehavior = SplitBehavior::Nothing) const; [[nodiscard]] Vector split_view(StringView, SplitBehavior = SplitBehavior::Nothing) const; +#ifndef KERNEL [[nodiscard]] Vector split_view_if(Function const& predicate, SplitBehavior = SplitBehavior::Nothing) const; +#endif [[nodiscard]] StringView find_last_split_view(char separator) const { @@ -239,7 +247,11 @@ class StringView { No, Yes, }; + +#ifndef KERNEL [[nodiscard]] Vector lines(ConsiderCarriageReturn = ConsiderCarriageReturn::Yes) const; +#endif + [[nodiscard]] size_t count_lines(ConsiderCarriageReturn = ConsiderCarriageReturn::Yes) const; // Create a new substring view of this string view, starting either at the beginning of diff --git a/AK/Vector.h b/AK/Vector.h index 3f15fdc42e3115..0d529fd2b64d1f 100644 --- a/AK/Vector.h +++ b/AK/Vector.h @@ -58,6 +58,9 @@ requires(!IsRvalueReference) class Vector { { } + constexpr static auto InlineCapacity = inline_capacity; + +#ifndef KERNEL Vector(std::initializer_list list) requires(!IsLvalueReference) { @@ -65,6 +68,7 @@ requires(!IsRvalueReference) class Vector { for (auto& item : list) unchecked_append(item); } +#endif Vector(Vector&& other) : m_size(other.m_size) @@ -82,6 +86,12 @@ requires(!IsRvalueReference) class Vector { other.reset_capacity(); } +#ifdef KERNEL + Vector(Vector const&) = delete; + + template + Vector(Vector const&) = delete; +#else Vector(Vector const& other) { ensure_capacity(other.size()); @@ -104,6 +114,29 @@ requires(!IsRvalueReference) class Vector { TypedTransfer::copy(data(), other.data(), other.size()); m_size = other.size(); } +#endif + + ErrorOr clone() const + { + Vector new_vector; + + TRY(new_vector.try_ensure_capacity(size())); + TypedTransfer::copy(new_vector.data(), data(), size()); + new_vector.m_size = size(); + + return new_vector; + } + + static ErrorOr from_span(ReadonlySpan other) + { + Vector new_vector; + + TRY(new_vector.try_ensure_capacity(other.size())); + TypedTransfer::copy(new_vector.data(), other.data(), other.size()); + new_vector.m_size = other.size(); + + return new_vector; + } ~Vector() { @@ -271,8 +304,6 @@ requires(!IsRvalueReference) class Vector { MUST(try_extend(other)); } -#endif - ALWAYS_INLINE void append(T&& value) { if constexpr (contains_reference) @@ -287,7 +318,6 @@ requires(!IsRvalueReference) class Vector { MUST(try_append(T(value))); } -#ifndef KERNEL void append(StorageType const* values, size_t count) { MUST(try_append(values, count)); @@ -365,6 +395,12 @@ requires(!IsRvalueReference) class Vector { return *this; } +#ifdef KERNEL + Vector& operator=(Vector const&) = delete; + + template + Vector& operator=(Vector const&) = delete; +#else Vector& operator=(Vector const& other) { if (this != &other) { @@ -385,6 +421,7 @@ requires(!IsRvalueReference) class Vector { m_size = other.size(); return *this; } +#endif void clear() { @@ -716,6 +753,7 @@ requires(!IsRvalueReference) class Vector { return try_resize(new_size, true); } +#ifndef KERNEL void grow_capacity(size_t needed_capacity) { MUST(try_grow_capacity(needed_capacity)); @@ -725,6 +763,7 @@ requires(!IsRvalueReference) class Vector { { MUST(try_ensure_capacity(needed_capacity)); } +#endif void shrink(size_t new_size, bool keep_capacity = false) { @@ -745,6 +784,7 @@ requires(!IsRvalueReference) class Vector { m_size = new_size; } +#ifndef KERNEL void resize(size_t new_size, bool keep_capacity = false) requires(!contains_reference) { @@ -768,6 +808,7 @@ requires(!IsRvalueReference) class Vector { } *this = move(new_vector); } +#endif using ConstIterator = SimpleIterator; using Iterator = SimpleIterator; diff --git a/Base/home/anon/.config/Tests.ini b/Base/home/anon/.config/Tests.ini index ea0e04b5462c61..1d0364b20038fb 100644 --- a/Base/home/anon/.config/Tests.ini +++ b/Base/home/anon/.config/Tests.ini @@ -1,6 +1,6 @@ [Global] SkipDirectories=Kernel/Legacy Shell -SkipRegex=^ue-.*$ +SkipRegex=^$ SkipTests=TestCommonmark function.sh NotTestsPattern=^.*(txt|frm|inc|so|elf)$ diff --git a/Base/home/anon/Tests/run-tests-and-shutdown.sh b/Base/home/anon/Tests/run-tests-and-shutdown.sh index 62d72ede4b1eb5..a031d69ef6ea8e 100755 --- a/Base/home/anon/Tests/run-tests-and-shutdown.sh +++ b/Base/home/anon/Tests/run-tests-and-shutdown.sh @@ -5,10 +5,11 @@ echo echo "==== Running Tests on SerenityOS ====" echo "architecture is: >>$(uname -m)<<" -if [ "$(uname -m)" = "AArch64" ] && [ "$1" != "--force" ] { +if [ "$(uname -m)" = "AArch64" -o "$(uname -m)" = "riscv64" ] && [ "$1" != "--force" ] { fail_count=0 } else { + mkdir -p "$HOME/profiles" export LLVM_PROFILE_FILE="$HOME/profiles/%p-profile.profraw" run-tests --show-progress=false --unlink-coredumps fail_count=$? diff --git a/Base/usr/share/man/man1/config.md b/Base/usr/share/man/man1/config.md index c0ee0c21568338..cbeb45cef4ab03 100644 --- a/Base/usr/share/man/man1/config.md +++ b/Base/usr/share/man/man1/config.md @@ -5,7 +5,7 @@ config ## Synopsis ```sh -$ config [--remove] [key] [value] +$ config [--remove] [group] [key] [value] ``` ## Description diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 68324ad6a53c51..e402ff457ac74f 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -93,6 +93,7 @@ Ping them right away if it's something urgent! If it's less urgent, advertise yo - [@BertalanD](https://github.com/BertalanD) - [@GMTA](https://github.com/gmta) - [@Lubrsi](https://github.com/Lubrsi) +- [@LucasChollet](https://github.com/LucasChollet) - [@nico](https://github.com/nico) - [@spholz](https://github.com/spholz) - [@timschumi](https://github.com/timschumi) diff --git a/Kernel/Arch/Processor.cpp b/Kernel/Arch/Processor.cpp index 5ff5eedcfa5a24..8ddb7381584b95 100644 --- a/Kernel/Arch/Processor.cpp +++ b/Kernel/Arch/Processor.cpp @@ -19,8 +19,7 @@ namespace Kernel { READONLY_AFTER_INIT FPUState s_clean_fpu_state; READONLY_AFTER_INIT Atomic g_total_processors; -template -void ProcessorBase::check_invoke_scheduler() +void ProcessorBase::check_invoke_scheduler() { VERIFY_INTERRUPTS_DISABLED(); VERIFY(!m_in_irq); @@ -31,10 +30,8 @@ void ProcessorBase::check_invoke_scheduler() Scheduler::invoke_async(); } } -template void ProcessorBase::check_invoke_scheduler(); -template -void ProcessorBase::deferred_call_queue(Function callback) +void ProcessorBase::deferred_call_queue(Function callback) { // NOTE: If we are called outside of a critical section and outside // of an irq handler, the function will be executed before we return! @@ -46,19 +43,17 @@ void ProcessorBase::deferred_call_queue(Function callback) cur_proc.m_deferred_call_pool.queue_entry(entry); } -template void ProcessorBase::deferred_call_queue(Function); -template -void ProcessorBase::enter_trap(TrapFrame& trap, bool raise_irq) +void ProcessorBase::enter_trap(TrapFrame& trap, bool raise_irq) { VERIFY_INTERRUPTS_DISABLED(); VERIFY(&Processor::current() == this); -#if ARCH(X86_64) - // FIXME: Figure out if we need prev_irq_level - trap.prev_irq_level = m_in_irq; -#endif + + // m_in_irq is always <= 1 since nested interrupts don't happen + // (because we never re-enable interrupts during interrupt handling). if (raise_irq) - m_in_irq++; + m_in_irq = 1; + auto* current_thread = Processor::current_thread(); if (current_thread) { auto& current_trap = current_thread->current_trap(); @@ -72,25 +67,32 @@ void ProcessorBase::enter_trap(TrapFrame& trap, bool raise_irq) trap.next_trap = nullptr; } } -template void ProcessorBase::enter_trap(TrapFrame&, bool); -template -u64 ProcessorBase::time_spent_idle() const +InterruptsState ProcessorBase::interrupts_state() +{ + return ProcessorBase::are_interrupts_enabled() ? InterruptsState::Enabled : InterruptsState::Disabled; +} + +void ProcessorBase::restore_interrupts_state(InterruptsState interrupts_state) +{ + if (interrupts_state == InterruptsState::Enabled) + ProcessorBase::enable_interrupts(); + else + ProcessorBase::disable_interrupts(); +} + +u64 ProcessorBase::time_spent_idle() const { return m_idle_thread->time_in_user() + m_idle_thread->time_in_kernel(); } -template u64 ProcessorBase::time_spent_idle() const; -template -void ProcessorBase::leave_critical() +void ProcessorBase::leave_critical() { InterruptDisabler disabler; current().do_leave_critical(); } -template void ProcessorBase::leave_critical(); -template -void ProcessorBase::do_leave_critical() +void ProcessorBase::do_leave_critical() { VERIFY(m_in_critical > 0); if (m_in_critical == 1) { @@ -105,7 +107,6 @@ void ProcessorBase::do_leave_critical() m_in_critical = m_in_critical - 1; } } -template void ProcessorBase::do_leave_critical(); void exit_kernel_thread(void) { @@ -135,8 +136,7 @@ void do_context_first_init(Thread* from_thread, Thread* to_thread) Scheduler::leave_on_first_switch(InterruptsState::Disabled); } -template -ErrorOr> ProcessorBase::capture_stack_trace(Thread& thread, size_t max_frames) +ErrorOr> ProcessorBase::capture_stack_trace(Thread& thread, size_t max_frames) { FlatPtr frame_ptr = 0, pc = 0; Vector stack_trace; @@ -264,6 +264,64 @@ ErrorOr> ProcessorBase::capture_stack_trace(Thread& threa return stack_trace; } -template ErrorOr> ProcessorBase::capture_stack_trace(Thread&, size_t); + +void ProcessorBase::exit_trap(TrapFrame& trap) +{ + VERIFY_INTERRUPTS_DISABLED(); + VERIFY(&Processor::current() == this); + + // Temporarily enter a critical section. This is to prevent critical + // sections entered and left within e.g. smp_process_pending_messages + // to trigger a context switch while we're executing this function + // See the comment at the end of the function why we don't use + // ScopedCritical here. + m_in_critical = m_in_critical + 1; + + m_in_irq = 0; + +#if ARCH(X86_64) + auto* self = static_cast(this); + if (is_smp_enabled()) + self->smp_process_pending_messages(); +#endif + + auto* current_thread = Processor::current_thread(); + if (current_thread) { + SpinlockLocker thread_lock(current_thread->get_lock()); + current_thread->check_dispatch_pending_signal(YieldBehavior::FlagYield); + } + + // Process the deferred call queue. Among other things, this ensures + // that any pending thread unblocks happen before we enter the scheduler. + m_deferred_call_pool.execute_pending(); + + if (current_thread) { + auto& current_trap = current_thread->current_trap(); + current_trap = trap.next_trap; + ExecutionMode new_previous_mode; + if (current_trap) { + VERIFY(current_trap->regs); + // If we have another higher level trap then we probably returned + // from an interrupt or irq handler. + new_previous_mode = current_trap->regs->previous_mode(); + } else { + // If we don't have a higher level trap then we're back in user mode. + // Which means that the previous mode prior to being back in user mode was kernel mode + new_previous_mode = ExecutionMode::Kernel; + } + + if (current_thread->set_previous_mode(new_previous_mode)) + current_thread->update_time_scheduled(TimeManagement::scheduler_current_time(), true, false); + } + + VERIFY_INTERRUPTS_DISABLED(); + + // Leave the critical section without actually enabling interrupts. + // We don't want context switches to happen until we're explicitly + // triggering a switch in check_invoke_scheduler. + m_in_critical = m_in_critical - 1; + if (!m_in_irq && !m_in_critical) + check_invoke_scheduler(); +} } diff --git a/Kernel/Arch/Processor.h b/Kernel/Arch/Processor.h index 21acc48d7310e2..343cad2b09c094 100644 --- a/Kernel/Arch/Processor.h +++ b/Kernel/Arch/Processor.h @@ -48,7 +48,6 @@ extern "C" void thread_context_first_enter(void); extern "C" void do_assume_context(Thread* thread, u32 flags); extern "C" FlatPtr do_init_context(Thread* thread, u32) __attribute__((used)); -template class ProcessorBase { public: template @@ -76,11 +75,11 @@ class ProcessorBase { ALWAYS_INLINE static bool is_initialized(); [[noreturn]] static void halt(); - void wait_for_interrupt() const; + void idle() const; ALWAYS_INLINE static void pause(); ALWAYS_INLINE static void wait_check(); - ALWAYS_INLINE static ProcessorT& current(); + ALWAYS_INLINE static Processor& current(); static Processor& by_id(u32); ALWAYS_INLINE u32 id() const @@ -94,7 +93,10 @@ class ProcessorBase { } ALWAYS_INLINE static u32 current_id(); - ALWAYS_INLINE static bool is_bootstrap_processor(); + ALWAYS_INLINE static bool is_bootstrap_processor() + { + return current_id() == 0; + } ALWAYS_INLINE bool has_nx() const; ALWAYS_INLINE bool has_feature(CPUFeature::Type const& feature) const { @@ -174,7 +176,7 @@ class ProcessorBase { static ErrorOr> capture_stack_trace(Thread& thread, size_t max_frames = 0); protected: - ProcessorT* m_self; + Processor* m_self; CPUFeature::Type m_features; Atomic m_halt_requested; @@ -203,7 +205,7 @@ class ProcessorBase { DeferredCallPool m_deferred_call_pool {}; }; -template class ProcessorBase; +static_assert(!IsPolymorphic); // We don't want runtime polymorphism in such a low-level class. } @@ -219,27 +221,6 @@ template class ProcessorBase; namespace Kernel { -template -ALWAYS_INLINE bool ProcessorBase::is_bootstrap_processor() -{ - return current_id() == 0; -} - -template -InterruptsState ProcessorBase::interrupts_state() -{ - return Processor::are_interrupts_enabled() ? InterruptsState::Enabled : InterruptsState::Disabled; -} - -template -void ProcessorBase::restore_interrupts_state(InterruptsState interrupts_state) -{ - if (interrupts_state == InterruptsState::Enabled) - Processor::enable_interrupts(); - else - Processor::disable_interrupts(); -} - struct ProcessorMessageEntry; struct ProcessorMessage { using CallbackFunction = Function; diff --git a/Kernel/Arch/ProcessorFunctions.include b/Kernel/Arch/ProcessorFunctions.include deleted file mode 100644 index 2544789bc3b77f..00000000000000 --- a/Kernel/Arch/ProcessorFunctions.include +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2023, kleines Filmröllchen - * - * SPDX-License-Identifier: BSD-2-Clause - */ - -#pragma once - -#include - -// This header instantiates all functions of ProcessorBase that are architecture-specific. -namespace Kernel { -template bool ProcessorBase::is_smp_enabled(); -template void ProcessorBase::idle_begin() const; -template void ProcessorBase::idle_end() const; -template void ProcessorBase::smp_enable(); -template void ProcessorBase::flush_tlb_local(VirtualAddress vaddr, size_t page_count); -template void ProcessorBase::flush_entire_tlb_local(); -template void ProcessorBase::flush_tlb(Memory::PageDirectory const*, VirtualAddress, size_t); -template void ProcessorBase::flush_instruction_cache(VirtualAddress vaddr, size_t byte_count); -template void ProcessorBase::early_initialize(u32 cpu); -template void ProcessorBase::initialize(u32 cpu); -template void ProcessorBase::halt(); -template void ProcessorBase::exit_trap(TrapFrame& trap); -template u32 ProcessorBase::clear_critical(); -template bool ProcessorBase::are_interrupts_enabled(); -template void ProcessorBase::wait_for_interrupt() const; -template Processor& ProcessorBase::by_id(u32 id); -template StringView ProcessorBase::platform_string(); -template void ProcessorBase::initialize_context_switching(Thread& initial_thread); -template void ProcessorBase::switch_context(Thread*& from_thread, Thread*& to_thread); -template void ProcessorBase::assume_context(Thread& thread, InterruptsState new_interrupts_state); -template FlatPtr ProcessorBase::init_context(Thread& thread, bool leave_crit); -template u32 ProcessorBase::smp_wake_n_idle_processors(u32 wake_count); -template void ProcessorBase::store_fpu_state(FPUState&); -template void ProcessorBase::load_fpu_state(FPUState const&); -} diff --git a/Kernel/Arch/TrapFrame.h b/Kernel/Arch/TrapFrame.h index f60b66715c00f8..9758c45d76e789 100644 --- a/Kernel/Arch/TrapFrame.h +++ b/Kernel/Arch/TrapFrame.h @@ -7,26 +7,27 @@ #pragma once #include - -// FIXME: There's only a minor difference between x86 and Aarch64/RISC-V trap frames; the prev_irq member. -// This seems to be unnecessary (see FIXME in Processor::enter_trap), -// so investigate whether we need it and either: -// (1) Remove the member and corresponding code from x86 -// (2) Implement prev_irq in the assembly stubs of Aarch64 and RISC-V -// and then use the same TrapFrame on all architectures. - -#if ARCH(X86_64) -# include -#elif ARCH(AARCH64) -# include -#elif ARCH(RISCV64) -# include -#else -# error "Unknown architecture" -#endif +#include namespace Kernel { +struct RegisterState; + +struct TrapFrame { + TrapFrame* next_trap; + RegisterState* regs; // must be last + + TrapFrame() = delete; + TrapFrame(TrapFrame const&) = delete; + TrapFrame(TrapFrame&&) = delete; + TrapFrame& operator=(TrapFrame const&) = delete; + TrapFrame& operator=(TrapFrame&&) = delete; +}; + +#define TRAP_FRAME_SIZE (2 * 8) + +static_assert(AssertSize()); + extern "C" void enter_trap_no_irq(TrapFrame* trap) __attribute__((used)); extern "C" void enter_trap(TrapFrame*) __attribute__((used)); extern "C" void exit_trap(TrapFrame*) __attribute__((used)); diff --git a/Kernel/Arch/aarch64/Delay.cpp b/Kernel/Arch/aarch64/Delay.cpp index fc0e92617e0c17..abbc820c605bc8 100644 --- a/Kernel/Arch/aarch64/Delay.cpp +++ b/Kernel/Arch/aarch64/Delay.cpp @@ -6,14 +6,15 @@ #include #include +#include namespace Kernel { void microseconds_delay(u32 microseconds) { - auto frequency = Aarch64::CNTFRQ_EL0::read().ClockFrequency; + VERIFY(ARMv8Timer::is_initialized()); - // TODO: Fall back to the devicetree clock-frequency property. + auto frequency = ARMv8Timer::the().ticks_per_second(); VERIFY(frequency != 0); // Use the EL1 virtual timer, as that timer should should be accessible to us both on device and in a VM. diff --git a/Kernel/Arch/aarch64/InterruptManagement.cpp b/Kernel/Arch/aarch64/InterruptManagement.cpp index 7116f13645727e..a50202db3bf546 100644 --- a/Kernel/Arch/aarch64/InterruptManagement.cpp +++ b/Kernel/Arch/aarch64/InterruptManagement.cpp @@ -7,12 +7,11 @@ #include #include -#include #include namespace Kernel { -static NeverDestroyed>>> s_recipes; +static NeverDestroyed>> s_interrupt_controllers; static InterruptManagement* s_interrupt_management; bool InterruptManagement::initialized() @@ -31,28 +30,17 @@ void InterruptManagement::initialize() VERIFY(!InterruptManagement::initialized()); s_interrupt_management = new InterruptManagement; - the().find_controllers(); + if (s_interrupt_controllers->is_empty()) + PANIC("InterruptManagement: No supported interrupt controller was found"); } -void InterruptManagement::add_recipe(DeviceTree::DeviceRecipe> recipe) +ErrorOr InterruptManagement::register_interrupt_controller(NonnullLockRefPtr interrupt_controller) { - s_recipes->append(move(recipe)); -} - -void InterruptManagement::find_controllers() -{ - for (auto& recipe : *s_recipes) { - auto device_or_error = recipe.create_device(); - if (device_or_error.is_error()) { - dmesgln("InterruptManagement: Failed to create interrupt controller for device \"{}\" with driver {}: {}", recipe.node_name, recipe.driver_name, device_or_error.release_error()); - continue; - } - - m_interrupt_controllers.append(device_or_error.release_value()); - } + // This function has to be called before InterruptManagement is initialized, + // as we do not support dynamic registration of interrupt controllers. + VERIFY(!initialized()); - if (m_interrupt_controllers.is_empty()) - PANIC("InterruptManagement: No supported interrupt controller found in devicetree"); + return s_interrupt_controllers->try_append(move(interrupt_controller)); } u8 InterruptManagement::acquire_mapped_interrupt_number(u8 interrupt_number) @@ -62,14 +50,14 @@ u8 InterruptManagement::acquire_mapped_interrupt_number(u8 interrupt_number) Vector> const& InterruptManagement::controllers() { - return m_interrupt_controllers; + return *s_interrupt_controllers; } NonnullLockRefPtr InterruptManagement::get_responsible_irq_controller(u8) { // TODO: Support more interrupt controllers - VERIFY(m_interrupt_controllers.size() == 1); - return m_interrupt_controllers[0]; + VERIFY(s_interrupt_controllers->size() == 1); + return (*s_interrupt_controllers)[0]; } void InterruptManagement::enumerate_interrupt_handlers(Function) diff --git a/Kernel/Arch/aarch64/InterruptManagement.h b/Kernel/Arch/aarch64/InterruptManagement.h index 4cf572a58b45f0..4f1165645cf7d4 100644 --- a/Kernel/Arch/aarch64/InterruptManagement.h +++ b/Kernel/Arch/aarch64/InterruptManagement.h @@ -8,7 +8,6 @@ #include #include -#include #include namespace Kernel { @@ -19,7 +18,7 @@ class InterruptManagement { static void initialize(); static bool initialized(); - static void add_recipe(DeviceTree::DeviceRecipe>); + static ErrorOr register_interrupt_controller(NonnullLockRefPtr); static u8 acquire_mapped_interrupt_number(u8 original_irq); @@ -30,9 +29,6 @@ class InterruptManagement { private: InterruptManagement() = default; - void find_controllers(); - - Vector> m_interrupt_controllers; }; } diff --git a/Kernel/Arch/aarch64/Interrupts/GIC.cpp b/Kernel/Arch/aarch64/Interrupts/GIC.cpp index 6d44380ff88c50..f2d80524b776cc 100644 --- a/Kernel/Arch/aarch64/Interrupts/GIC.cpp +++ b/Kernel/Arch/aarch64/Interrupts/GIC.cpp @@ -132,6 +132,46 @@ Optional GIC::pending_interrupt() const return interrupt_number; } +ErrorOr GIC::translate_interrupt_specifier_to_interrupt_number(ReadonlyBytes interrupt_specifier) const +{ + // https://www.kernel.org/doc/Documentation/devicetree/bindings/interrupt-controller/arm,gic.yaml + + if (interrupt_specifier.size() != 3 * sizeof(u32)) + return EINVAL; + + FixedMemoryStream stream { interrupt_specifier }; + + enum class InterruptType : u32 { + SPI = 0, + PPI = 1, + }; + + auto interrupt_type = MUST(stream.read_value>()); + auto interrupt_number = MUST(stream.read_value>()); + auto flags = MUST(stream.read_value>()); + + (void)flags; // FIXME: Use this to configure the trigger mode properly. + + // Section 2.2.1 Interrupt IDs + if (interrupt_type == InterruptType::SPI) { + // "Interrupt numbers ID32-ID1019 are used for SPIs." + if (interrupt_number + 32 > 1019) + return ERANGE; + + return interrupt_number + 32; + } + + if (interrupt_type == InterruptType::PPI) { + // "ID16-ID31 are used for PPIs" + if (interrupt_number + 16 > 31) + return ERANGE; + + return interrupt_number + 16; + } + + return EINVAL; +} + UNMAP_AFTER_INIT GIC::GIC(Memory::TypedMapping distributor_registers, Memory::TypedMapping cpu_interface_registers) : m_distributor_registers(move(distributor_registers)) , m_cpu_interface_registers(move(cpu_interface_registers)) @@ -184,7 +224,7 @@ static constinit Array const compatibles_array = { "arm,cortex-a15-gic"sv, }; -EARLY_DEVICETREE_DRIVER(GICDriver, compatibles_array); +INTERRUPT_CONTROLLER_DEVICETREE_DRIVER(GICDriver, compatibles_array); // https://www.kernel.org/doc/Documentation/devicetree/bindings/interrupt-controller/arm,gic.yaml ErrorOr GICDriver::probe(DeviceTree::Device const& device, StringView) const @@ -192,15 +232,10 @@ ErrorOr GICDriver::probe(DeviceTree::Device const& device, StringView) con auto distributor_registers_resource = TRY(device.get_resource(0)); auto cpu_interface_registers_resource = TRY(device.get_resource(1)); - DeviceTree::DeviceRecipe> recipe { - name(), - device.node_name(), - [distributor_registers_resource, cpu_interface_registers_resource] { - return GIC::try_to_initialize(distributor_registers_resource, cpu_interface_registers_resource); - }, - }; + auto gic = TRY(GIC::try_to_initialize(distributor_registers_resource, cpu_interface_registers_resource)); - InterruptManagement::add_recipe(move(recipe)); + MUST(DeviceTree::Management::register_interrupt_controller(device, *gic)); + MUST(InterruptManagement::register_interrupt_controller(move(gic))); return {}; } diff --git a/Kernel/Arch/aarch64/Interrupts/GIC.h b/Kernel/Arch/aarch64/Interrupts/GIC.h index 8297011ee3dd61..e6431826b0a08a 100644 --- a/Kernel/Arch/aarch64/Interrupts/GIC.h +++ b/Kernel/Arch/aarch64/Interrupts/GIC.h @@ -8,6 +8,7 @@ #include #include +#include #include // Only GICv2 is currently supported. @@ -16,7 +17,9 @@ namespace Kernel { -class GIC final : public IRQController { +class GIC final + : public IRQController + , public DeviceTree::InterruptController { public: static ErrorOr> try_to_initialize(DeviceTree::Device::Resource distributor_registers_resource, DeviceTree::Device::Resource cpu_interface_registers_registers_resource); @@ -29,6 +32,9 @@ class GIC final : public IRQController { virtual StringView model() const override { return "GIC"sv; } + // ^DeviceTree::InterruptController + virtual ErrorOr translate_interrupt_specifier_to_interrupt_number(ReadonlyBytes) const override; + struct DistributorRegisters; struct CPUInterfaceRegisters; diff --git a/Kernel/Arch/aarch64/Processor.cpp b/Kernel/Arch/aarch64/Processor.cpp index 3715662b0ec42c..e2e0ae79df0d53 100644 --- a/Kernel/Arch/aarch64/Processor.cpp +++ b/Kernel/Arch/aarch64/Processor.cpp @@ -30,20 +30,17 @@ extern "C" void enter_thread_context(Thread* from_thread, Thread* to_thread) __a Processor* g_current_processor; -template -void ProcessorBase::store_fpu_state(FPUState& fpu_state) +void ProcessorBase::store_fpu_state(FPUState& fpu_state) { ::store_fpu_state(&fpu_state); } -template -void ProcessorBase::load_fpu_state(FPUState const& fpu_state) +void ProcessorBase::load_fpu_state(FPUState const& fpu_state) { ::load_fpu_state(&fpu_state); } -template -void ProcessorBase::early_initialize(u32 cpu) +void ProcessorBase::early_initialize(u32 cpu) { VERIFY(g_current_processor == nullptr); m_cpu = cpu; @@ -54,8 +51,7 @@ void ProcessorBase::early_initialize(u32 cpu) g_current_processor = static_cast(this); } -template -void ProcessorBase::initialize(u32) +void ProcessorBase::initialize(u32) { m_deferred_call_pool.init(); @@ -75,16 +71,14 @@ void ProcessorBase::initialize(u32) initialize_interrupts(); } -template -[[noreturn]] void ProcessorBase::halt() +[[noreturn]] void ProcessorBase::halt() { disable_interrupts(); for (;;) asm volatile("wfi"); } -template -void ProcessorBase::flush_tlb_local(VirtualAddress, size_t) +void ProcessorBase::flush_tlb_local(VirtualAddress, size_t) { // FIXME: Figure out how to flush a single page asm volatile("dsb ishst"); @@ -93,8 +87,7 @@ void ProcessorBase::flush_tlb_local(VirtualAddress, size_t) asm volatile("isb"); } -template -void ProcessorBase::flush_entire_tlb_local() +void ProcessorBase::flush_entire_tlb_local() { asm volatile("dsb ishst"); asm volatile("tlbi vmalle1"); @@ -102,20 +95,17 @@ void ProcessorBase::flush_entire_tlb_local() asm volatile("isb"); } -template -void ProcessorBase::flush_tlb(Memory::PageDirectory const*, VirtualAddress vaddr, size_t page_count) +void ProcessorBase::flush_tlb(Memory::PageDirectory const*, VirtualAddress vaddr, size_t page_count) { flush_tlb_local(vaddr, page_count); } -template -void ProcessorBase::flush_instruction_cache(VirtualAddress vaddr, size_t byte_count) +void ProcessorBase::flush_instruction_cache(VirtualAddress vaddr, size_t byte_count) { __builtin___clear_cache(reinterpret_cast(vaddr.as_ptr()), reinterpret_cast(vaddr.offset(byte_count).as_ptr())); } -template -u32 ProcessorBase::clear_critical() +u32 ProcessorBase::clear_critical() { InterruptDisabler disabler; auto prev_critical = in_critical(); @@ -126,16 +116,14 @@ u32 ProcessorBase::clear_critical() return prev_critical; } -template -u32 ProcessorBase::smp_wake_n_idle_processors(u32 wake_count) +u32 ProcessorBase::smp_wake_n_idle_processors(u32 wake_count) { (void)wake_count; // FIXME: Actually wake up other cores when SMP is supported for aarch64. return 0; } -template -void ProcessorBase::initialize_context_switching(Thread& initial_thread) +void ProcessorBase::initialize_context_switching(Thread& initial_thread) { VERIFY(initial_thread.process().is_kernel_process()); @@ -162,8 +150,7 @@ void ProcessorBase::initialize_context_switching(Thread& initial_thread) VERIFY_NOT_REACHED(); } -template -void ProcessorBase::switch_context(Thread*& from_thread, Thread*& to_thread) +void ProcessorBase::switch_context(Thread*& from_thread, Thread*& to_thread) { VERIFY(!m_in_irq); VERIFY(m_in_critical == 1); @@ -269,8 +256,7 @@ extern "C" FlatPtr do_init_context(Thread* thread, u32 new_interrupts_state) } // FIXME: Share this code with other architectures. -template -void ProcessorBase::assume_context(Thread& thread, InterruptsState new_interrupts_state) +void ProcessorBase::assume_context(Thread& thread, InterruptsState new_interrupts_state) { dbgln_if(CONTEXT_SWITCH_DEBUG, "Assume context for thread {} {}", VirtualAddress(&thread), thread); @@ -285,8 +271,7 @@ void ProcessorBase::assume_context(Thread& thread, InterruptsState new_interr VERIFY_NOT_REACHED(); } -template -FlatPtr ProcessorBase::init_context(Thread& thread, bool leave_crit) +FlatPtr ProcessorBase::init_context(Thread& thread, bool leave_crit) { VERIFY(g_scheduler_lock.is_locked()); if (leave_crit) { @@ -346,55 +331,6 @@ FlatPtr ProcessorBase::init_context(Thread& thread, bool leave_crit) return stack_top; } -// FIXME: Figure out if we can fully share this code with x86. -template -void ProcessorBase::exit_trap(TrapFrame& trap) -{ - VERIFY_INTERRUPTS_DISABLED(); - VERIFY(&Processor::current() == this); - - // Temporarily enter a critical section. This is to prevent critical - // sections entered and left within e.g. smp_process_pending_messages - // to trigger a context switch while we're executing this function - // See the comment at the end of the function why we don't use - // ScopedCritical here. - m_in_critical = m_in_critical + 1; - - // FIXME: Figure out if we need prev_irq_level, see duplicated code in Kernel/Arch/x86/common/Processor.cpp - m_in_irq = 0; - - // Process the deferred call queue. Among other things, this ensures - // that any pending thread unblocks happen before we enter the scheduler. - m_deferred_call_pool.execute_pending(); - - auto* current_thread = Processor::current_thread(); - if (current_thread) { - auto& current_trap = current_thread->current_trap(); - current_trap = trap.next_trap; - ExecutionMode new_previous_mode; - if (current_trap) { - VERIFY(current_trap->regs); - new_previous_mode = current_trap->regs->previous_mode(); - } else { - // If we don't have a higher level trap then we're back in user mode. - // Which means that the previous mode prior to being back in user mode was kernel mode - new_previous_mode = ExecutionMode::Kernel; - } - - if (current_thread->set_previous_mode(new_previous_mode)) - current_thread->update_time_scheduled(TimeManagement::scheduler_current_time(), true, false); - } - - VERIFY_INTERRUPTS_DISABLED(); - - // Leave the critical section without actually enabling interrupts. - // We don't want context switches to happen until we're explicitly - // triggering a switch in check_invoke_scheduler. - m_in_critical = m_in_critical - 1; - if (!m_in_irq && !m_in_critical) - check_invoke_scheduler(); -} - NAKED void thread_context_first_enter(void) { asm( @@ -464,25 +400,52 @@ extern "C" void enter_thread_context(Thread* from_thread, Thread* to_thread) } } -template -StringView ProcessorBase::platform_string() +StringView ProcessorBase::platform_string() { return "aarch64"sv; } -template -void ProcessorBase::wait_for_interrupt() const +void ProcessorBase::idle() const { - asm("wfi"); + VERIFY_INTERRUPTS_DISABLED(); + + idle_begin(); + asm volatile(R"( + // Go to sleep. Execution will continue when the processor receives an interrupt, even with interrupts disabled. + // The processor may also be woken up by other implementation defined wake events, in that case we return from this function without an interrupt. + wfi + + // Shortly unmask interrupts to call the interrupt handler for the received interrupt. + msr daifclr, #2 // PSTATE.I = 0 + msr daifset, #2 // PSTATE.I = 1 + )" :); + idle_end(); } -template -Processor& ProcessorBase::by_id(u32 id) +Processor& ProcessorBase::by_id(u32 id) { (void)id; TODO_AARCH64(); } +void ProcessorBase::idle_begin() const +{ + // FIXME: Implement this when SMP for aarch64 is supported. } -#include +void ProcessorBase::idle_end() const +{ + // FIXME: Implement this when SMP for aarch64 is supported. +} + +void ProcessorBase::smp_enable() +{ + // FIXME: Implement this when SMP for aarch64 is supported. +} + +bool ProcessorBase::is_smp_enabled() +{ + return false; +} + +} diff --git a/Kernel/Arch/aarch64/Processor.h b/Kernel/Arch/aarch64/Processor.h index 10a23cf81c3e4e..d34b7dbf6469c3 100644 --- a/Kernel/Arch/aarch64/Processor.h +++ b/Kernel/Arch/aarch64/Processor.h @@ -32,15 +32,12 @@ class Processor; struct TrapFrame; enum class InterruptsState; -template -class ProcessorBase; - // FIXME: Remove this once we support SMP in aarch64 extern Processor* g_current_processor; constexpr size_t MAX_CPU_COUNT = 1; -class Processor final : public ProcessorBase { +class Processor final : public ProcessorBase { public: template Callback> static inline IterationDecision for_each(Callback callback) @@ -60,144 +57,101 @@ class Processor final : public ProcessorBase { } }; -template -ALWAYS_INLINE bool ProcessorBase::is_initialized() +ALWAYS_INLINE bool ProcessorBase::is_initialized() { return g_current_processor != nullptr; } -template -ALWAYS_INLINE Thread* ProcessorBase::idle_thread() +ALWAYS_INLINE Thread* ProcessorBase::idle_thread() { return current().m_idle_thread; } -template -ALWAYS_INLINE void ProcessorBase::set_current_thread(Thread& current_thread) +ALWAYS_INLINE void ProcessorBase::set_current_thread(Thread& current_thread) { current().m_current_thread = ¤t_thread; } // FIXME: When aarch64 supports multiple cores, return the correct core id here. -template -ALWAYS_INLINE u32 ProcessorBase::current_id() +ALWAYS_INLINE u32 ProcessorBase::current_id() { return 0; } -template -ALWAYS_INLINE u32 ProcessorBase::in_critical() +ALWAYS_INLINE u32 ProcessorBase::in_critical() { return current().m_in_critical; } -template -ALWAYS_INLINE void ProcessorBase::enter_critical() +ALWAYS_INLINE void ProcessorBase::enter_critical() { auto& current_processor = current(); current_processor.m_in_critical = current_processor.m_in_critical + 1; } -template -ALWAYS_INLINE void ProcessorBase::restore_critical(u32 prev_critical) +ALWAYS_INLINE void ProcessorBase::restore_critical(u32 prev_critical) { current().m_in_critical = prev_critical; } -template -ALWAYS_INLINE T& ProcessorBase::current() +ALWAYS_INLINE Processor& ProcessorBase::current() { return *g_current_processor; } -template -void ProcessorBase::idle_begin() const -{ - // FIXME: Implement this when SMP for aarch64 is supported. -} - -template -void ProcessorBase::idle_end() const -{ - // FIXME: Implement this when SMP for aarch64 is supported. -} - -template -void ProcessorBase::smp_enable() -{ - // FIXME: Implement this when SMP for aarch64 is supported. -} - -template -bool ProcessorBase::is_smp_enabled() -{ - return false; -} - -template -ALWAYS_INLINE bool ProcessorBase::are_interrupts_enabled() +ALWAYS_INLINE bool ProcessorBase::are_interrupts_enabled() { auto daif = Aarch64::DAIF::read(); return !daif.I; } -template -ALWAYS_INLINE void ProcessorBase::enable_interrupts() +ALWAYS_INLINE void ProcessorBase::enable_interrupts() { Aarch64::DAIF::clear_I(); } -template -ALWAYS_INLINE void ProcessorBase::disable_interrupts() +ALWAYS_INLINE void ProcessorBase::disable_interrupts() { Aarch64::DAIF::set_I(); } -template -ALWAYS_INLINE bool ProcessorBase::current_in_scheduler() +ALWAYS_INLINE bool ProcessorBase::current_in_scheduler() { return current().m_in_scheduler; } -template -ALWAYS_INLINE void ProcessorBase::set_current_in_scheduler(bool value) +ALWAYS_INLINE void ProcessorBase::set_current_in_scheduler(bool value) { current().m_in_scheduler = value; } -template -ALWAYS_INLINE bool ProcessorBase::has_nx() const +ALWAYS_INLINE bool ProcessorBase::has_nx() const { return true; } -template -ALWAYS_INLINE FlatPtr ProcessorBase::current_in_irq() +ALWAYS_INLINE FlatPtr ProcessorBase::current_in_irq() { return current().m_in_irq; } -template -ALWAYS_INLINE Thread* ProcessorBase::current_thread() +ALWAYS_INLINE Thread* ProcessorBase::current_thread() { return current().m_current_thread; } -template -ALWAYS_INLINE void ProcessorBase::pause() +ALWAYS_INLINE void ProcessorBase::pause() { asm volatile("isb sy"); } -template -ALWAYS_INLINE void ProcessorBase::wait_check() +ALWAYS_INLINE void ProcessorBase::wait_check() { asm volatile("yield"); // FIXME: Process SMP messages once we support SMP on aarch64; cf. x86_64 } -template -ALWAYS_INLINE Optional ProcessorBase::read_cycle_count() +ALWAYS_INLINE Optional ProcessorBase::read_cycle_count() { if (Processor::current().has_feature(CPUFeature::PMUv3)) return Aarch64::PMCCNTR_EL0::read().CCNT; diff --git a/Kernel/Arch/aarch64/RPi/InterruptController.cpp b/Kernel/Arch/aarch64/RPi/InterruptController.cpp index fd7fe9bdf81476..8d6e0435236e11 100644 --- a/Kernel/Arch/aarch64/RPi/InterruptController.cpp +++ b/Kernel/Arch/aarch64/RPi/InterruptController.cpp @@ -72,27 +72,64 @@ Optional InterruptController::pending_interrupt() const return irq_number_plus_one - 1; } +ErrorOr InterruptController::translate_interrupt_specifier_to_interrupt_number(ReadonlyBytes interrupt_specifier) const +{ + // https://www.kernel.org/doc/Documentation/devicetree/bindings/interrupt-controller/brcm,bcm2835-armctrl-ic.txt + + if (interrupt_specifier.size() != 2 * sizeof(u32)) + return EINVAL; + + FixedMemoryStream stream { interrupt_specifier }; + + enum class InterruptBank : u32 { + BasicPendingRegister = 0, + GPUPendingRegister1 = 1, + GPUPendingRegister2 = 2, + }; + + auto interrupt_bank = MUST(stream.read_value>()); + auto interrupt_number = MUST(stream.read_value>()); + + if (interrupt_bank == InterruptBank::BasicPendingRegister) { + dbgln("FIXME: Support interrupts in the BCM2835 basic pending register"); + return ENOTSUP; + } + + if (interrupt_bank == InterruptBank::GPUPendingRegister1) { + // We map interrupts in GPU pending register 1 to 0-31. + if (interrupt_number > 31) + return ERANGE; + + return interrupt_number; + } + + if (interrupt_bank == InterruptBank::GPUPendingRegister2) { + // We map interrupts in GPU pending register 2 to 31-63. + if (interrupt_number > 31) + return ERANGE; + + return interrupt_number + 32; + } + + return EINVAL; +} + static constinit Array const compatibles_array = { "brcm,bcm2836-armctrl-ic"sv, }; -EARLY_DEVICETREE_DRIVER(BCM2836InterruptControllerDriver, compatibles_array); +INTERRUPT_CONTROLLER_DEVICETREE_DRIVER(BCM2836InterruptControllerDriver, compatibles_array); // https://www.kernel.org/doc/Documentation/devicetree/bindings/interrupt-controller/brcm,bcm2835-armctrl-ic.txt ErrorOr BCM2836InterruptControllerDriver::probe(DeviceTree::Device const& device, StringView) const { auto physical_address = TRY(device.get_resource(0)).paddr; - DeviceTree::DeviceRecipe> recipe { - name(), - device.node_name(), - [physical_address]() -> ErrorOr> { - auto registers_mapping = TRY(Memory::map_typed_writable(physical_address)); - return adopt_nonnull_lock_ref_or_enomem(new (nothrow) InterruptController(move(registers_mapping))); - }, - }; + auto registers_mapping = TRY(Memory::map_typed_writable(physical_address)); + auto interrupt_controller = TRY(adopt_nonnull_lock_ref_or_enomem(new (nothrow) InterruptController(move(registers_mapping)))); - InterruptManagement::add_recipe(move(recipe)); + MUST(DeviceTree::Management::register_interrupt_controller(device, *interrupt_controller)); + MUST(InterruptManagement::register_interrupt_controller(move(interrupt_controller))); return {}; } diff --git a/Kernel/Arch/aarch64/RPi/InterruptController.h b/Kernel/Arch/aarch64/RPi/InterruptController.h index e67e240f3f74bf..c92357831e6cd8 100644 --- a/Kernel/Arch/aarch64/RPi/InterruptController.h +++ b/Kernel/Arch/aarch64/RPi/InterruptController.h @@ -9,6 +9,7 @@ #include #include #include +#include #include namespace Kernel::RPi { @@ -18,7 +19,9 @@ struct InterruptControllerRegisters; // This class implements the simple Interrupt Controller found in the BCM2837. (RPi3) // A description of this device can be found at chapter 7 (Interrupts) of the manual: // https://github.com/raspberrypi/documentation/files/1888662/BCM2837-ARM-Peripherals.-.Revised.-.V2-1.pdf (RPi3) -class InterruptController : public IRQController { +class InterruptController + : public IRQController + , public DeviceTree::InterruptController { public: InterruptController(Memory::TypedMapping); @@ -35,6 +38,9 @@ class InterruptController : public IRQController { return "Raspberry Pi Interrupt Controller"sv; } + // ^DeviceTree::InterruptController + virtual ErrorOr translate_interrupt_specifier_to_interrupt_number(ReadonlyBytes) const override; + Memory::TypedMapping m_registers; }; diff --git a/Kernel/Arch/aarch64/RPi/Timer.cpp b/Kernel/Arch/aarch64/RPi/Timer.cpp index 5662776cf95491..c1482d20149923 100644 --- a/Kernel/Arch/aarch64/RPi/Timer.cpp +++ b/Kernel/Arch/aarch64/RPi/Timer.cpp @@ -194,32 +194,15 @@ EARLY_DEVICETREE_DRIVER(BCM2835TimerDriver, compatibles_array); // https://www.kernel.org/doc/Documentation/devicetree/bindings/timer/brcm,bcm2835-system-timer.txt ErrorOr BCM2835TimerDriver::probe(DeviceTree::Device const& device, StringView) const { - auto const interrupts = TRY(device.node().interrupts(DeviceTree::get())); - if (interrupts.size() != 4) - return EINVAL; // The devicetree binding requires 4 interrupts. + auto physical_address = TRY(device.get_resource(0)).paddr; + auto registers_mapping = TRY(Memory::map_typed_writable(physical_address)); // This driver currently only uses channel 1. - auto const& interrupt = interrupts[1]; - - // FIXME: Don't depend on a specific interrupt descriptor format and implement proper devicetree interrupt mapping/translation. - if (!interrupt.domain_root->is_compatible_with("brcm,bcm2836-armctrl-ic"sv)) - return ENOTSUP; - if (interrupt.interrupt_identifier.size() != sizeof(BigEndian)) - return ENOTSUP; - auto const interrupt_number = *reinterpret_cast const*>(interrupt.interrupt_identifier.data()) & 0xffff'ffff; - - auto physical_address = TRY(device.get_resource(0)).paddr; + auto interrupt_number = TRY(device.get_interrupt_number(1)); - DeviceTree::DeviceRecipe> recipe { - name(), - device.node_name(), - [physical_address, interrupt_number]() -> ErrorOr> { - auto registers_mapping = TRY(Memory::map_typed_writable(physical_address)); - return adopt_nonnull_lock_ref_or_enomem(new (nothrow) Timer(move(registers_mapping), interrupt_number)); - }, - }; + auto timer = TRY(adopt_nonnull_lock_ref_or_enomem(new (nothrow) Timer(move(registers_mapping), interrupt_number))); - TimeManagement::add_recipe(move(recipe)); + MUST(TimeManagement::register_hardware_timer(move(timer))); return {}; } diff --git a/Kernel/Arch/aarch64/Time/ARMv8Timer.cpp b/Kernel/Arch/aarch64/Time/ARMv8Timer.cpp index 8e806b7fd50c74..eff95cd8b0b214 100644 --- a/Kernel/Arch/aarch64/Time/ARMv8Timer.cpp +++ b/Kernel/Arch/aarch64/Time/ARMv8Timer.cpp @@ -12,22 +12,20 @@ namespace Kernel { -ARMv8Timer::ARMv8Timer(u8 interrupt_number) +static ARMv8Timer* s_the = nullptr; + +ARMv8Timer::ARMv8Timer(u8 interrupt_number, u32 frequency) : HardwareTimer(interrupt_number) + , m_frequency(frequency) { - m_frequency = Aarch64::CNTFRQ_EL0::read().ClockFrequency; - - // TODO: Fall back to the devicetree clock-frequency property. - VERIFY(m_frequency != 0); - m_interrupt_interval = m_frequency / OPTIMAL_TICKS_PER_SECOND_RATE; start_timer(m_interrupt_interval); } -ErrorOr> ARMv8Timer::initialize(u8 interrupt_number) +ErrorOr> ARMv8Timer::initialize(u8 interrupt_number, u32 frequency) { - auto timer = TRY(adopt_nonnull_lock_ref_or_enomem(new (nothrow) ARMv8Timer(interrupt_number))); + auto timer = TRY(adopt_nonnull_lock_ref_or_enomem(new (nothrow) ARMv8Timer(interrupt_number, frequency))); // Enable the physical timer. auto ctl = Aarch64::CNTV_CTL_EL0::read(); @@ -40,6 +38,16 @@ ErrorOr> ARMv8Timer::initialize(u8 interrupt_numbe return timer; } +bool ARMv8Timer::is_initialized() +{ + return s_the != nullptr; +} + +ARMv8Timer& ARMv8Timer::the() +{ + return *s_the; +} + u64 ARMv8Timer::current_ticks() { return Aarch64::CNTVCT_EL0::read().VirtualCount; @@ -105,8 +113,6 @@ EARLY_DEVICETREE_DRIVER(ARMv8TimerDriver, compatibles_array); // https://www.kernel.org/doc/Documentation/devicetree/bindings/timer/arm,arch_timer.yaml ErrorOr ARMv8TimerDriver::probe(DeviceTree::Device const& device, StringView) const { - auto const interrupts = TRY(device.node().interrupts(DeviceTree::get())); - if (device.node().has_property("interrupt-names"sv)) return ENOTSUP; // TODO: Support the interrupt-names property. @@ -118,36 +124,32 @@ ErrorOr ARMv8TimerDriver::probe(DeviceTree::Device const& device, StringVi }; // Use the EL1 virtual timer, as that timer should should be accessible to us both on device and in a VM. - if (interrupts.size() < (to_underlying(DeviceTreeTimerInterruptIndex::EL1Virtual) + 1)) - return ENOTSUP; + auto interrupt_number = TRY(device.get_interrupt_number(to_underlying(DeviceTreeTimerInterruptIndex::EL1Virtual))); - auto const& interrupt = interrupts[to_underlying(DeviceTreeTimerInterruptIndex::EL1Virtual)]; + u32 frequency = 0; - // FIXME: Don't depend on a specific interrupt descriptor format and implement proper devicetree interrupt mapping/translation. - if (!interrupt.domain_root->is_compatible_with("arm,gic-400"sv) && !interrupt.domain_root->is_compatible_with("arm,cortex-a15-gic"sv)) - return ENOTSUP; - if (interrupt.interrupt_identifier.size() != 3 * sizeof(BigEndian)) - return ENOTSUP; + auto clock_frequency_property = device.node().get_property("clock-frequency"sv); + if (clock_frequency_property.has_value()) { + if (clock_frequency_property->size() != sizeof(u32)) { + dmesgln("ARMv8Timer: \"clock-frequency\" property for \"{}\" has invalid size: {}", device.node_name(), clock_frequency_property->size()); + return EINVAL; + } - // The ARM timer uses a PPI (Private Peripheral Interrupt). - // GIC interrupts 16-31 are for PPIs, so add 16 to get the GIC interrupt ID. + frequency = clock_frequency_property->as(); + } else { + frequency = Aarch64::CNTFRQ_EL0::read().ClockFrequency; + } - // The interrupt type is in the first cell. It should be 1 for PPIs. - if (reinterpret_cast const*>(interrupt.interrupt_identifier.data())[0] != 1) - return ENOTSUP; + if (frequency == 0) { + dmesgln("ARMv8Timer: Unable to determine clock frequency for \"{}\"", device.node_name()); + return EINVAL; + } - // The interrupt number is in the second cell. - auto interrupt_number = (reinterpret_cast const*>(interrupt.interrupt_identifier.data())[1]) + 16; + auto timer = TRY(ARMv8Timer::initialize(interrupt_number, frequency)); - DeviceTree::DeviceRecipe> recipe { - name(), - device.node_name(), - [interrupt_number] { - return ARMv8Timer::initialize(interrupt_number); - }, - }; + MUST(TimeManagement::register_hardware_timer(timer)); - TimeManagement::add_recipe(move(recipe)); + s_the = &timer.leak_ref(); return {}; } diff --git a/Kernel/Arch/aarch64/Time/ARMv8Timer.h b/Kernel/Arch/aarch64/Time/ARMv8Timer.h index 14348b77e0c87a..eea1dd14e86857 100644 --- a/Kernel/Arch/aarch64/Time/ARMv8Timer.h +++ b/Kernel/Arch/aarch64/Time/ARMv8Timer.h @@ -17,7 +17,10 @@ namespace Kernel { class ARMv8Timer final : public HardwareTimer { public: - static ErrorOr> initialize(u8 interrupt_number); + static ErrorOr> initialize(u8 interrupt_number, u32 frequency); + + static bool is_initialized(); + static ARMv8Timer& the(); virtual HardwareTimerType timer_type() const override { return HardwareTimerType::ARMv8Timer; } virtual StringView model() const override { return "ARMv8 Timer"sv; } @@ -38,7 +41,7 @@ class ARMv8Timer final : public HardwareTimer { u64 update_time(u64& seconds_since_boot, u32& ticks_this_second, bool query_only); private: - ARMv8Timer(u8 interrupt_number); + ARMv8Timer(u8 interrupt_number, u32 frequency); static u64 current_ticks(); static void start_timer(u32 delta); diff --git a/Kernel/Arch/aarch64/TrapFrame.h b/Kernel/Arch/aarch64/TrapFrame.h deleted file mode 100644 index a72986e0e1fdd1..00000000000000 --- a/Kernel/Arch/aarch64/TrapFrame.h +++ /dev/null @@ -1,31 +0,0 @@ -/* - * Copyright (c) 2018-2021, Andreas Kling - * Copyright (c) 2022, Gunnar Beutner - * - * SPDX-License-Identifier: BSD-2-Clause - */ - -#pragma once - -#include -#include -#include -#include - -namespace Kernel { - -struct TrapFrame { - TrapFrame* next_trap; - RegisterState* regs; - - TrapFrame() = delete; - TrapFrame(TrapFrame const&) = delete; - TrapFrame(TrapFrame&&) = delete; - TrapFrame& operator=(TrapFrame const&) = delete; - TrapFrame& operator=(TrapFrame&&) = delete; -}; - -#define TRAP_FRAME_SIZE (2 * 8) -static_assert(AssertSize()); - -} diff --git a/Kernel/Arch/init.cpp b/Kernel/Arch/init.cpp index d95ac78611f888..4c1bec815f401a 100644 --- a/Kernel/Arch/init.cpp +++ b/Kernel/Arch/init.cpp @@ -200,7 +200,7 @@ extern "C" [[noreturn]] UNMAP_AFTER_INIT NO_SANITIZE_COVERAGE void init(BootInfo } } dmesgln("Starting SerenityOS..."); - + dmesgln("Kernel Commandline: {}", kernel_command_line().string()); dmesgln("Boot method: {}", boot_info.boot_method); MM.unmap_prekernel(); @@ -224,14 +224,21 @@ extern "C" [[noreturn]] UNMAP_AFTER_INIT NO_SANITIZE_COVERAGE void init(BootInfo DeviceTree::dump_fdt(); DeviceTree::Management::initialize(); -#endif -#if ARCH(RISCV64) +# if ARCH(RISCV64) bsp_processor().find_and_parse_devicetree_node(); init_delay_loop(); +# endif + + MUST(DeviceTree::Management::the().probe_drivers(DeviceTree::Driver::ProbeStage::InterruptController)); #endif InterruptManagement::initialize(); + +#if ARCH(AARCH64) || ARCH(RISCV64) + MUST(DeviceTree::Management::the().probe_drivers(DeviceTree::Driver::ProbeStage::Early)); +#endif + ACPI::initialize(); // Initialize TimeManagement before using randomness! @@ -440,6 +447,10 @@ UNMAP_AFTER_INIT void setup_serial_debug() if (s_kernel_cmdline.contains("serial_debug"sv)) { set_serial_debug_enabled(true); } + + if (s_kernel_cmdline.contains("pci_serial_debug"sv)) { + set_pci_serial_debug_enabled(true); + } } // Define some Itanium C++ ABI methods to stop the linker from complaining. diff --git a/Kernel/Arch/riscv64/InterruptManagement.cpp b/Kernel/Arch/riscv64/InterruptManagement.cpp index 324e0971949981..7c5633fe51813a 100644 --- a/Kernel/Arch/riscv64/InterruptManagement.cpp +++ b/Kernel/Arch/riscv64/InterruptManagement.cpp @@ -6,15 +6,13 @@ #include #include -#include #include -#include -#include #include +#include namespace Kernel { -static NeverDestroyed>>> s_recipes; +static NeverDestroyed>> s_interrupt_controllers; static InterruptManagement* s_interrupt_management; bool InterruptManagement::initialized() @@ -33,28 +31,17 @@ void InterruptManagement::initialize() VERIFY(!InterruptManagement::initialized()); s_interrupt_management = new InterruptManagement; - the().find_controllers(); + if (s_interrupt_controllers->is_empty()) + PANIC("InterruptManagement: No supported interrupt controller was found"); } -void InterruptManagement::add_recipe(DeviceTree::DeviceRecipe> recipe) +ErrorOr InterruptManagement::register_interrupt_controller(NonnullLockRefPtr interrupt_controller) { - s_recipes->append(move(recipe)); -} - -void InterruptManagement::find_controllers() -{ - for (auto& recipe : *s_recipes) { - auto device_or_error = recipe.create_device(); - if (device_or_error.is_error()) { - dmesgln("InterruptManagement: Failed to create interrupt controller for device \"{}\" with driver {}: {}", recipe.node_name, recipe.driver_name, device_or_error.release_error()); - continue; - } - - m_interrupt_controllers.append(device_or_error.release_value()); - } + // This function has to be called before InterruptManagement is initialized, + // as we do not support dynamic registration of interrupt controllers. + VERIFY(!initialized()); - if (m_interrupt_controllers.is_empty()) - dmesgln("InterruptManagement: No supported interrupt controller found in devicetree"); + return s_interrupt_controllers->try_append(move(interrupt_controller)); } u8 InterruptManagement::acquire_mapped_interrupt_number(u8 original_irq) @@ -64,14 +51,14 @@ u8 InterruptManagement::acquire_mapped_interrupt_number(u8 original_irq) Vector> const& InterruptManagement::controllers() { - return m_interrupt_controllers; + return *s_interrupt_controllers; } NonnullLockRefPtr InterruptManagement::get_responsible_irq_controller(size_t) { // TODO: Support more interrupt controllers - VERIFY(m_interrupt_controllers.size() == 1); - return m_interrupt_controllers[0]; + VERIFY(s_interrupt_controllers->size() == 1); + return (*s_interrupt_controllers)[0]; } void InterruptManagement::enumerate_interrupt_handlers(Function callback) diff --git a/Kernel/Arch/riscv64/InterruptManagement.h b/Kernel/Arch/riscv64/InterruptManagement.h index 14e6cab3b27aed..2287c75291f7b8 100644 --- a/Kernel/Arch/riscv64/InterruptManagement.h +++ b/Kernel/Arch/riscv64/InterruptManagement.h @@ -8,7 +8,7 @@ #include #include -#include +#include #include VALIDATE_IS_RISCV64() @@ -21,7 +21,7 @@ class InterruptManagement { static void initialize(); static bool initialized(); - static void add_recipe(DeviceTree::DeviceRecipe>); + static ErrorOr register_interrupt_controller(NonnullLockRefPtr); static u8 acquire_mapped_interrupt_number(u8 original_irq); @@ -32,9 +32,6 @@ class InterruptManagement { private: InterruptManagement() = default; - void find_controllers(); - - Vector> m_interrupt_controllers; }; } diff --git a/Kernel/Arch/riscv64/Interrupts/PLIC.cpp b/Kernel/Arch/riscv64/Interrupts/PLIC.cpp index fd51b15a14f7c8..e70f9d5b7539a4 100644 --- a/Kernel/Arch/riscv64/Interrupts/PLIC.cpp +++ b/Kernel/Arch/riscv64/Interrupts/PLIC.cpp @@ -62,12 +62,30 @@ u8 PLIC::pending_interrupt() const return m_registers->contexts[m_boot_hart_supervisor_mode_context_id].claim_complete; } +ErrorOr PLIC::translate_interrupt_specifier_to_interrupt_number(ReadonlyBytes interrupt_specifier) const +{ + // https://www.kernel.org/doc/Documentation/devicetree/bindings/interrupt-controller/sifive,plic-1.0.0.yaml + // For generic PLICs the #interrupt-cells property should be 1 and the interrupt specifier should simply be the interrupt number. + + if (interrupt_specifier.size() != 1 * sizeof(u32)) + return EINVAL; + + FixedMemoryStream stream { interrupt_specifier }; + + auto interrupt_number = MUST(stream.read_value>()); + + if (interrupt_number >= m_interrupt_count) + return ERANGE; + + return interrupt_number; +} + static constinit Array const compatibles_array = { "riscv,plic0"sv, "sifive,plic-1.0.0"sv, }; -EARLY_DEVICETREE_DRIVER(PLICDriver, compatibles_array); +INTERRUPT_CONTROLLER_DEVICETREE_DRIVER(PLICDriver, compatibles_array); // https://www.kernel.org/doc/Documentation/devicetree/bindings/interrupt-controller/sifive,plic-1.0.0.yaml ErrorOr PLICDriver::probe(DeviceTree::Device const& device, StringView) const @@ -97,14 +115,13 @@ ErrorOr PLICDriver::probe(DeviceTree::Device const& device, StringView) co if (!cpu->is_compatible_with("riscv"sv)) return EINVAL; - u64 interrupt_specifier = 0; - if (interrupt.interrupt_identifier.size() == sizeof(u32)) - interrupt_specifier = *reinterpret_cast const*>(interrupt.interrupt_identifier.data()); - else if (interrupt.interrupt_identifier.size() == sizeof(u64)) - interrupt_specifier = *reinterpret_cast const*>(interrupt.interrupt_identifier.data()); - else + // https://www.kernel.org/doc/Documentation/devicetree/bindings/interrupt-controller/riscv,cpu-intc.yaml + // #interrupt-cells: "const: 1" + if (interrupt.interrupt_specifier.size() != sizeof(u32)) return EINVAL; + auto interrupt_specifier = *reinterpret_cast const*>(interrupt.interrupt_specifier.data()); + // https://www.kernel.org/doc/Documentation/devicetree/bindings/riscv/cpus.yaml // reg: "The hart ID of this CPU node." auto hart_id = TRY(TRY(TRY(cpu->reg()).entry(0)).bus_address().as_flatptr()); @@ -114,16 +131,11 @@ ErrorOr PLICDriver::probe(DeviceTree::Device const& device, StringView) co } } - DeviceTree::DeviceRecipe> recipe { - name(), - device.node_name(), - [physical_address, max_interrupt_id, boot_hart_supervisor_mode_context_id]() -> ErrorOr> { - auto registers_mapping = TRY(Memory::map_typed_writable(physical_address)); - return adopt_nonnull_lock_ref_or_enomem(new (nothrow) PLIC(move(registers_mapping), max_interrupt_id + 1, boot_hart_supervisor_mode_context_id)); - }, - }; + auto registers_mapping = TRY(Memory::map_typed_writable(physical_address)); + auto interrupt_controller = TRY(adopt_nonnull_lock_ref_or_enomem(new (nothrow) PLIC(move(registers_mapping), max_interrupt_id + 1, boot_hart_supervisor_mode_context_id))); - InterruptManagement::add_recipe(move(recipe)); + MUST(DeviceTree::Management::register_interrupt_controller(device, *interrupt_controller)); + MUST(InterruptManagement::register_interrupt_controller(move(interrupt_controller))); return {}; } diff --git a/Kernel/Arch/riscv64/Interrupts/PLIC.h b/Kernel/Arch/riscv64/Interrupts/PLIC.h index 48b3dce7ede823..cfaa212f116460 100644 --- a/Kernel/Arch/riscv64/Interrupts/PLIC.h +++ b/Kernel/Arch/riscv64/Interrupts/PLIC.h @@ -7,11 +7,14 @@ #pragma once #include +#include #include namespace Kernel { -class PLIC final : public IRQController { +class PLIC final + : public IRQController + , public DeviceTree::InterruptController { public: struct RegisterMap { u32 interrupt_priority[1024]; @@ -38,6 +41,9 @@ class PLIC final : public IRQController { virtual StringView model() const override { return "PLIC"sv; } + // ^DeviceTree::InterruptController + virtual ErrorOr translate_interrupt_specifier_to_interrupt_number(ReadonlyBytes) const override; + private: void initialize(); diff --git a/Kernel/Arch/riscv64/Processor.cpp b/Kernel/Arch/riscv64/Processor.cpp index 4f7f92e8f4165c..4e2ec494323db2 100644 --- a/Kernel/Arch/riscv64/Processor.cpp +++ b/Kernel/Arch/riscv64/Processor.cpp @@ -98,8 +98,7 @@ static void load_vector_state(FPUState const& fpu_state) [vtype] "r"(fpu_state.vtype)); } -template -void ProcessorBase::store_fpu_state(FPUState& fpu_state) +void ProcessorBase::store_fpu_state(FPUState& fpu_state) { asm volatile( "fsd f0, 0*8(%0) \n" @@ -143,8 +142,7 @@ void ProcessorBase::store_fpu_state(FPUState& fpu_state) store_vector_state(fpu_state); } -template -void ProcessorBase::load_fpu_state(FPUState const& fpu_state) +void ProcessorBase::load_fpu_state(FPUState const& fpu_state) { asm volatile( "fld f0, 0*8(%0) \n" @@ -188,8 +186,7 @@ void ProcessorBase::load_fpu_state(FPUState const& fpu_state) load_vector_state(fpu_state); } -template -void ProcessorBase::early_initialize(u32 cpu) +void ProcessorBase::early_initialize(u32 cpu) { VERIFY(g_current_processor == nullptr); m_cpu = cpu; @@ -197,8 +194,7 @@ void ProcessorBase::early_initialize(u32 cpu) g_current_processor = static_cast(this); } -template -void ProcessorBase::initialize(u32) +void ProcessorBase::initialize(u32) { m_deferred_call_pool.init(); @@ -214,8 +210,7 @@ void ProcessorBase::initialize(u32) initialize_interrupts(); } -template -[[noreturn]] void ProcessorBase::halt() +[[noreturn]] void ProcessorBase::halt() { // WFI ignores the value of sstatus.SIE, so we can't use disable_interrupts(). // Instead, disable all interrupts sources by setting sie to zero. @@ -224,8 +219,7 @@ template asm volatile("wfi"); } -template -void ProcessorBase::flush_tlb_local(VirtualAddress vaddr, size_t page_count) +void ProcessorBase::flush_tlb_local(VirtualAddress vaddr, size_t page_count) { auto addr = vaddr.get(); while (page_count > 0) { @@ -238,28 +232,24 @@ void ProcessorBase::flush_tlb_local(VirtualAddress vaddr, size_t page_count) } } -template -void ProcessorBase::flush_entire_tlb_local() +void ProcessorBase::flush_entire_tlb_local() { asm volatile("sfence.vma"); } -template -void ProcessorBase::flush_tlb(Memory::PageDirectory const*, VirtualAddress vaddr, size_t page_count) +void ProcessorBase::flush_tlb(Memory::PageDirectory const*, VirtualAddress vaddr, size_t page_count) { // FIXME: Use the SBI RFENCE extension to flush the TLB of other harts when we support SMP on riscv64. flush_tlb_local(vaddr, page_count); } -template -void ProcessorBase::flush_instruction_cache(VirtualAddress, size_t) +void ProcessorBase::flush_instruction_cache(VirtualAddress, size_t) { // FIXME: Use the SBI RFENCE extension to flush the instruction cache of other harts when we support SMP on riscv64. asm volatile("fence.i" ::: "memory"); } -template -u32 ProcessorBase::clear_critical() +u32 ProcessorBase::clear_critical() { InterruptDisabler disabler; auto prev_critical = in_critical(); @@ -270,15 +260,13 @@ u32 ProcessorBase::clear_critical() return prev_critical; } -template -u32 ProcessorBase::smp_wake_n_idle_processors(u32) +u32 ProcessorBase::smp_wake_n_idle_processors(u32) { // FIXME: Actually wake up other cores when SMP is supported for riscv64. return 0; } -template -void ProcessorBase::initialize_context_switching(Thread& initial_thread) +void ProcessorBase::initialize_context_switching(Thread& initial_thread) { VERIFY(initial_thread.process().is_kernel_process()); @@ -303,8 +291,7 @@ void ProcessorBase::initialize_context_switching(Thread& initial_thread) VERIFY_NOT_REACHED(); } -template -void ProcessorBase::switch_context(Thread*& from_thread, Thread*& to_thread) +void ProcessorBase::switch_context(Thread*& from_thread, Thread*& to_thread) { VERIFY(!m_in_irq); VERIFY(m_in_critical == 1); @@ -448,8 +435,7 @@ extern "C" FlatPtr do_init_context(Thread* thread, u32 new_interrupts_state) return Processor::current().init_context(*thread, true); } -template -void ProcessorBase::assume_context(Thread& thread, InterruptsState new_interrupts_state) +void ProcessorBase::assume_context(Thread& thread, InterruptsState new_interrupts_state) { dbgln_if(CONTEXT_SWITCH_DEBUG, "Assume context for thread {} {}", VirtualAddress(&thread), thread); @@ -464,8 +450,7 @@ void ProcessorBase::assume_context(Thread& thread, InterruptsState new_interr VERIFY_NOT_REACHED(); } -template -FlatPtr ProcessorBase::init_context(Thread& thread, bool leave_crit) +FlatPtr ProcessorBase::init_context(Thread& thread, bool leave_crit) { VERIFY(g_scheduler_lock.is_locked()); if (leave_crit) { @@ -524,53 +509,24 @@ FlatPtr ProcessorBase::init_context(Thread& thread, bool leave_crit) return stack_top; } -// FIXME: Figure out if we can fully share this code with x86. -template -void ProcessorBase::exit_trap(TrapFrame& trap) +void ProcessorBase::idle_begin() const { - VERIFY_INTERRUPTS_DISABLED(); - VERIFY(&Processor::current() == this); - - // Temporarily enter a critical section. This is to prevent critical - // sections entered and left within e.g. smp_process_pending_messages - // to trigger a context switch while we're executing this function - // See the comment at the end of the function why we don't use - // ScopedCritical here. - m_in_critical = m_in_critical + 1; - - // FIXME: Figure out if we need prev_irq_level, see duplicated code in Kernel/Arch/x86/common/Processor.cpp - m_in_irq = 0; - - // Process the deferred call queue. Among other things, this ensures - // that any pending thread unblocks happen before we enter the scheduler. - m_deferred_call_pool.execute_pending(); - - auto* current_thread = Processor::current_thread(); - if (current_thread) { - auto& current_trap = current_thread->current_trap(); - current_trap = trap.next_trap; - ExecutionMode new_previous_mode; - if (current_trap) { - VERIFY(current_trap->regs); - new_previous_mode = current_trap->regs->previous_mode(); - } else { - // If we don't have a higher level trap then we're back in user mode. - // Which means that the previous mode prior to being back in user mode was kernel mode - new_previous_mode = ExecutionMode::Kernel; - } + // FIXME: Implement this when SMP for riscv64 is supported. +} - if (current_thread->set_previous_mode(new_previous_mode)) - current_thread->update_time_scheduled(TimeManagement::scheduler_current_time(), true, false); - } +void ProcessorBase::idle_end() const +{ + // FIXME: Implement this when SMP for riscv64 is supported. +} - VERIFY_INTERRUPTS_DISABLED(); +void ProcessorBase::smp_enable() +{ + // FIXME: Implement this when SMP for riscv64 is supported. +} - // Leave the critical section without actually enabling interrupts. - // We don't want context switches to happen until we're explicitly - // triggering a switch in check_invoke_scheduler. - m_in_critical = m_in_critical - 1; - if (!m_in_irq && !m_in_critical) - check_invoke_scheduler(); +bool ProcessorBase::is_smp_enabled() +{ + return false; } extern "C" void context_first_init(Thread* from_thread, Thread* to_thread); @@ -637,20 +593,30 @@ NAKED void do_assume_context(Thread*, u32) // clang-format on } -template -StringView ProcessorBase::platform_string() +StringView ProcessorBase::platform_string() { return "riscv64"sv; } -template -void ProcessorBase::wait_for_interrupt() const +void ProcessorBase::idle() const { - asm("wfi"); + VERIFY_INTERRUPTS_DISABLED(); + + idle_begin(); + asm volatile(R"( + # Go to sleep. Execution will continue when the processor receives an interrupt, even with interrupts disabled. + # WFI is allowed to cause spurious wakeups, so implementing it as a NOP is legal. + # In that case the idle loop is constantly busy waiting for interrupts. + wfi + + # Shortly enable interrupts to call the interrupt handler for the received interrupt. + csrsi sstatus, 1 << 1 # sstatus.SIE = 1 + csrci sstatus, 1 << 1 # sstatus.SIE = 0 + )" :); + idle_end(); } -template -Processor& ProcessorBase::by_id(u32) +Processor& ProcessorBase::by_id(u32) { TODO_RISCV64(); } @@ -738,5 +704,3 @@ void Processor::generate_userspace_extension_bitmask() } } - -#include diff --git a/Kernel/Arch/riscv64/Processor.h b/Kernel/Arch/riscv64/Processor.h index a9a4299fb3d5ef..9a865de6c9ba8f 100644 --- a/Kernel/Arch/riscv64/Processor.h +++ b/Kernel/Arch/riscv64/Processor.h @@ -33,16 +33,13 @@ class Processor; struct TrapFrame; enum class InterruptsState; -template -class ProcessorBase; - // FIXME: Remove this once we support SMP in riscv64 extern Processor* g_current_processor; constexpr size_t MAX_CPU_COUNT = 1; -class Processor final : public ProcessorBase { - friend class ProcessorBase; +class Processor final : public ProcessorBase { + friend class ProcessorBase; public: template Callback> @@ -79,130 +76,89 @@ class Processor final : public ProcessorBase { Array m_userspace_extension_bitmask {}; }; -template -ALWAYS_INLINE bool ProcessorBase::is_initialized() +ALWAYS_INLINE bool ProcessorBase::is_initialized() { return g_current_processor != nullptr; } -template -ALWAYS_INLINE Thread* ProcessorBase::idle_thread() +ALWAYS_INLINE Thread* ProcessorBase::idle_thread() { return current().m_idle_thread; } -template -ALWAYS_INLINE void ProcessorBase::set_current_thread(Thread& current_thread) +ALWAYS_INLINE void ProcessorBase::set_current_thread(Thread& current_thread) { current().m_current_thread = ¤t_thread; } // FIXME: When riscv64 supports multiple cores, return the correct core id here. -template -ALWAYS_INLINE u32 ProcessorBase::current_id() +ALWAYS_INLINE u32 ProcessorBase::current_id() { return 0; } -template -ALWAYS_INLINE u32 ProcessorBase::in_critical() +ALWAYS_INLINE u32 ProcessorBase::in_critical() { return current().m_in_critical; } -template -ALWAYS_INLINE void ProcessorBase::enter_critical() +ALWAYS_INLINE void ProcessorBase::enter_critical() { auto& current_processor = current(); current_processor.m_in_critical += 1; } -template -ALWAYS_INLINE void ProcessorBase::restore_critical(u32 prev_critical) +ALWAYS_INLINE void ProcessorBase::restore_critical(u32 prev_critical) { current().m_in_critical = prev_critical; } -template -ALWAYS_INLINE T& ProcessorBase::current() +ALWAYS_INLINE Processor& ProcessorBase::current() { return *g_current_processor; } -template -void ProcessorBase::idle_begin() const -{ - // FIXME: Implement this when SMP for riscv64 is supported. -} - -template -void ProcessorBase::idle_end() const -{ - // FIXME: Implement this when SMP for riscv64 is supported. -} - -template -void ProcessorBase::smp_enable() -{ - // FIXME: Implement this when SMP for riscv64 is supported. -} - -template -bool ProcessorBase::is_smp_enabled() -{ - return false; -} - -template -ALWAYS_INLINE bool ProcessorBase::are_interrupts_enabled() +ALWAYS_INLINE bool ProcessorBase::are_interrupts_enabled() { return RISCV64::CSR::SSTATUS::read().SIE == 1; } -template -ALWAYS_INLINE void ProcessorBase::enable_interrupts() +ALWAYS_INLINE void ProcessorBase::enable_interrupts() { RISCV64::CSR::set_bits(RISCV64::CSR::SSTATUS::Bit::SIE); } -template -ALWAYS_INLINE void ProcessorBase::disable_interrupts() +ALWAYS_INLINE void ProcessorBase::disable_interrupts() { RISCV64::CSR::clear_bits(RISCV64::CSR::SSTATUS::Bit::SIE); } -template -ALWAYS_INLINE bool ProcessorBase::current_in_scheduler() +ALWAYS_INLINE bool ProcessorBase::current_in_scheduler() { return current().m_in_scheduler; } -template -ALWAYS_INLINE void ProcessorBase::set_current_in_scheduler(bool value) +ALWAYS_INLINE void ProcessorBase::set_current_in_scheduler(bool value) { current().m_in_scheduler = value; } -template -ALWAYS_INLINE bool ProcessorBase::has_nx() const +ALWAYS_INLINE bool ProcessorBase::has_nx() const { return true; } -template -ALWAYS_INLINE FlatPtr ProcessorBase::current_in_irq() +ALWAYS_INLINE FlatPtr ProcessorBase::current_in_irq() { return current().m_in_irq; } -template -ALWAYS_INLINE Thread* ProcessorBase::current_thread() +ALWAYS_INLINE Thread* ProcessorBase::current_thread() { return current().m_current_thread; } -template -ALWAYS_INLINE void ProcessorBase::pause() +ALWAYS_INLINE void ProcessorBase::pause() { // PAUSE is a HINT defined by the Zihintpause extension. // We don't have to check if that extension is supported, since HINTs effectively behave like NOPs if they are not implemented. @@ -214,15 +170,13 @@ ALWAYS_INLINE void ProcessorBase::pause() )" :); } -template -ALWAYS_INLINE void ProcessorBase::wait_check() +ALWAYS_INLINE void ProcessorBase::wait_check() { Processor::pause(); // FIXME: Process SMP messages once we support SMP on riscv64; cf. x86_64 } -template -ALWAYS_INLINE Optional ProcessorBase::read_cycle_count() +ALWAYS_INLINE Optional ProcessorBase::read_cycle_count() { return RISCV64::CSR::read(); } diff --git a/Kernel/Arch/riscv64/TrapFrame.h b/Kernel/Arch/riscv64/TrapFrame.h deleted file mode 100644 index 9ccdb3909014f8..00000000000000 --- a/Kernel/Arch/riscv64/TrapFrame.h +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Copyright (c) 2023, Sönke Holz - * - * SPDX-License-Identifier: BSD-2-Clause - */ - -#pragma once - -#include - -#include -#include - -VALIDATE_IS_RISCV64() - -namespace Kernel { - -struct TrapFrame { - TrapFrame* next_trap; - RegisterState* regs; - - TrapFrame() = delete; - TrapFrame(TrapFrame const&) = delete; - TrapFrame(TrapFrame&&) = delete; - TrapFrame& operator=(TrapFrame const&) = delete; - TrapFrame& operator=(TrapFrame&&) = delete; -}; - -#define TRAP_FRAME_SIZE (2 * 8) -static_assert(AssertSize()); - -extern "C" void exit_trap(TrapFrame*) __attribute__((used)); - -} diff --git a/Kernel/Arch/x86_64/Firmware/ACPI.cpp b/Kernel/Arch/x86_64/Firmware/ACPI.cpp index 8279810345b608..df5fafd49057aa 100644 --- a/Kernel/Arch/x86_64/Firmware/ACPI.cpp +++ b/Kernel/Arch/x86_64/Firmware/ACPI.cpp @@ -6,25 +6,77 @@ #include #include +#include #include namespace Kernel::ACPI::StaticParsing { +static bool is_rsdp_valid(u8 const* rsdp, size_t region_size) +{ + if (region_size < sizeof(Structures::RSDPDescriptor)) + return false; + + u8 revision = reinterpret_cast(rsdp)->revision; + + u8 checksum = 0; + for (size_t i = 0; i < sizeof(Structures::RSDPDescriptor); ++i) + checksum += rsdp[i]; + + if (checksum != 0) + return false; + + if (revision == 0) + // Checksum matched and there's nothing more to check. + return true; + + if (region_size < sizeof(Structures::RSDPDescriptor20)) + return false; + + checksum = 0; + for (size_t i = 0; i < sizeof(Structures::RSDPDescriptor20); ++i) + checksum += rsdp[i]; + return checksum == 0; +} + // https://uefi.org/specs/ACPI/6.5/05_ACPI_Software_Programming_Model.html#finding-the-rsdp-on-ia-pc-systems Optional find_rsdp_in_ia_pc_specific_memory_locations() { constexpr auto signature = "RSD PTR "sv; + + auto locate_rsdp = [&signature](Memory::MappedROM mapping) -> Optional { + constexpr size_t rsdp_alignment = 16; + size_t start_paddr = mapping.paddr.get() + mapping.offset; + size_t alignment_offset = align_up_to(start_paddr, rsdp_alignment) - start_paddr; + + mapping.offset += alignment_offset; + mapping.size = align_down_to(mapping.size - alignment_offset, rsdp_alignment); + + VERIFY((mapping.paddr.get() + mapping.offset) % rsdp_alignment == 0); + VERIFY(mapping.size % rsdp_alignment == 0); + + while (true) { + u8 const* rsdp = mapping.pointer_to_chunk_starting_with(signature, rsdp_alignment); + if (!rsdp) + return {}; + if (is_rsdp_valid(rsdp, static_cast(mapping.end() - rsdp))) + return mapping.paddr_of(rsdp); + size_t mapping_offset = static_cast(rsdp - mapping.base()) + rsdp_alignment; + mapping.offset += mapping_offset; + mapping.size -= mapping_offset; + } + }; + auto ebda_or_error = map_ebda(); if (!ebda_or_error.is_error()) { - auto rsdp = ebda_or_error.value().find_chunk_starting_with(signature, 16); - if (rsdp.has_value()) - return rsdp; + auto maybe_rsdp = locate_rsdp(ebda_or_error.release_value()); + if (maybe_rsdp.has_value()) + return maybe_rsdp.value(); } auto bios_or_error = map_bios(); if (!bios_or_error.is_error()) { - auto rsdp = bios_or_error.value().find_chunk_starting_with(signature, 16); - if (rsdp.has_value()) - return rsdp; + auto maybe_rsdp = locate_rsdp(bios_or_error.release_value()); + if (maybe_rsdp.has_value()) + return maybe_rsdp.value(); } // On some systems the RSDP may be located in ACPI NVS or reclaimable memory regions @@ -33,7 +85,7 @@ Optional find_rsdp_in_ia_pc_specific_memory_locations() if (!(memory_range.type == Memory::PhysicalMemoryRangeType::ACPI_NVS || memory_range.type == Memory::PhysicalMemoryRangeType::ACPI_Reclaimable)) return IterationDecision::Continue; - potential_ranges.append(memory_range); + potential_ranges.try_append(memory_range).release_value_but_fixme_should_propagate_errors(); return IterationDecision::Continue; }); @@ -52,9 +104,9 @@ Optional find_rsdp_in_ia_pc_specific_memory_locations() mapping.size = memory_range.length; mapping.paddr = memory_range.start; - auto rsdp = mapping.find_chunk_starting_with(signature, 16); - if (rsdp.has_value()) - return rsdp; + auto maybe_rsdp = locate_rsdp(move(mapping)); + if (maybe_rsdp.has_value()) + return maybe_rsdp.value(); } return {}; diff --git a/Kernel/Arch/x86_64/InterruptEntry.cpp b/Kernel/Arch/x86_64/InterruptEntry.cpp index 7c91ce38c5aaae..eab16a93d702e3 100644 --- a/Kernel/Arch/x86_64/InterruptEntry.cpp +++ b/Kernel/Arch/x86_64/InterruptEntry.cpp @@ -5,8 +5,8 @@ * SPDX-License-Identifier: BSD-2-Clause */ +#include #include -#include // clang-format off asm( diff --git a/Kernel/Arch/x86_64/InterruptManagement.cpp b/Kernel/Arch/x86_64/InterruptManagement.cpp index 643dc8376009d4..a205776015d6ca 100644 --- a/Kernel/Arch/x86_64/InterruptManagement.cpp +++ b/Kernel/Arch/x86_64/InterruptManagement.cpp @@ -137,7 +137,7 @@ UNMAP_AFTER_INIT void InterruptManagement::switch_to_pic_mode() VERIFY(m_interrupt_controllers.is_empty()); dmesgln("Interrupts: Switch to Legacy PIC mode"); InterruptDisabler disabler; - m_interrupt_controllers.append(adopt_lock_ref(*new PIC())); + m_interrupt_controllers.try_append(adopt_lock_ref(*new PIC())).release_value_but_fixme_should_propagate_errors(); SpuriousInterruptHandler::initialize(7); SpuriousInterruptHandler::initialize(15); dbgln("Interrupts: Detected {}", m_interrupt_controllers[0]->model()); @@ -186,7 +186,7 @@ UNMAP_AFTER_INIT void InterruptManagement::locate_apic_data() auto madt = Memory::map_typed(m_madt_physical_address.value()).release_value_but_fixme_should_propagate_errors(); if (madt->flags & PCAT_COMPAT_FLAG) - m_interrupt_controllers.append(adopt_lock_ref(*new PIC())); + m_interrupt_controllers.try_append(adopt_lock_ref(*new PIC())).release_value_but_fixme_should_propagate_errors(); size_t entry_index = 0; size_t entries_length = madt->h.length - sizeof(ACPI::Structures::MADT); auto* madt_entry = madt->entries; @@ -195,7 +195,7 @@ UNMAP_AFTER_INIT void InterruptManagement::locate_apic_data() if (madt_entry->type == (u8)ACPI::Structures::MADTEntryType::IOAPIC) { auto* ioapic_entry = (const ACPI::Structures::MADTEntries::IOAPIC*)madt_entry; dbgln("IOAPIC found @ MADT entry {}, MMIO Registers @ {}", entry_index, PhysicalAddress(ioapic_entry->ioapic_address)); - m_interrupt_controllers.append(adopt_lock_ref(*new IOAPIC(PhysicalAddress(ioapic_entry->ioapic_address), ioapic_entry->gsi_base))); + m_interrupt_controllers.try_append(adopt_lock_ref(*new IOAPIC(PhysicalAddress(ioapic_entry->ioapic_address), ioapic_entry->gsi_base))).release_value_but_fixme_should_propagate_errors(); } if (madt_entry->type == (u8)ACPI::Structures::MADTEntryType::InterruptSourceOverride) { auto* interrupt_override_entry = (const ACPI::Structures::MADTEntries::InterruptSourceOverride*)madt_entry; diff --git a/Kernel/Arch/x86_64/Interrupts.cpp b/Kernel/Arch/x86_64/Interrupts.cpp index cb976723775d84..b0ad19be2c78c4 100644 --- a/Kernel/Arch/x86_64/Interrupts.cpp +++ b/Kernel/Arch/x86_64/Interrupts.cpp @@ -77,13 +77,11 @@ static EntropySource s_entropy_source_interrupts { EntropySource::Static::Interr " pushq %rdi\n" \ " pushq %rsp \n" /* set TrapFrame::regs */ \ " subq $" __STRINGIFY(TRAP_FRAME_SIZE - 8) ", %rsp \n" \ - " subq $0x8, %rsp\n" /* align stack */ \ - " lea 0x8(%rsp), %rdi \n" \ + " movq %rsp, %rdi \n" \ " cld\n" \ " call enter_trap_no_irq \n" \ - " lea 0x8(%rsp), %rdi \n" \ + " movq %rsp, %rdi \n" \ " call " #title "_handler\n" \ - " addq $0x8, %rsp\n" /* undo alignment */ \ " jmp common_trap_exit \n" \ ); \ } diff --git a/Kernel/Arch/x86_64/Interrupts/APIC.cpp b/Kernel/Arch/x86_64/Interrupts/APIC.cpp index 9bc40be87bce70..be84a4e8bd124f 100644 --- a/Kernel/Arch/x86_64/Interrupts/APIC.cpp +++ b/Kernel/Arch/x86_64/Interrupts/APIC.cpp @@ -339,7 +339,7 @@ UNMAP_AFTER_INIT void APIC::setup_ap_boot_environment() memcpy(apic_startup_region_ptr, reinterpret_cast(apic_ap_start), apic_ap_start_size); // Allocate enough stacks for all APs - m_ap_temporary_boot_stacks.ensure_capacity(aps_to_enable); + m_ap_temporary_boot_stacks.try_ensure_capacity(aps_to_enable).release_value_but_fixme_should_propagate_errors(); for (u32 i = 0; i < aps_to_enable; i++) { auto stack_region_or_error = MM.allocate_kernel_region(Thread::default_kernel_stack_size, {}, Memory::Region::Access::ReadWrite, AllocationStrategy::AllocateNow); if (stack_region_or_error.is_error()) { @@ -360,7 +360,7 @@ UNMAP_AFTER_INIT void APIC::setup_ap_boot_environment() } // Allocate Processor structures for all APs and store the pointer to the data - m_ap_processor_info.resize(aps_to_enable); + m_ap_processor_info.try_resize(aps_to_enable).release_value_but_fixme_should_propagate_errors(); for (size_t i = 0; i < aps_to_enable; i++) m_ap_processor_info[i] = adopt_nonnull_own_or_enomem(new (nothrow) Processor()).release_value_but_fixme_should_propagate_errors(); auto* ap_processor_info_array = &ap_stack_array[aps_to_enable]; @@ -399,7 +399,7 @@ UNMAP_AFTER_INIT void APIC::do_boot_aps() // because we won't be able to send FlushTLB messages, so we have to // have all memory set up for the threads so that when the APs are // starting up, they can access all the memory properly - m_ap_idle_threads.resize(aps_to_enable); + m_ap_idle_threads.try_resize(aps_to_enable).release_value_but_fixme_should_propagate_errors(); for (u32 i = 0; i < aps_to_enable; i++) m_ap_idle_threads[i] = Scheduler::create_ap_idle_thread(i + 1); diff --git a/Kernel/Arch/x86_64/Processor.cpp b/Kernel/Arch/x86_64/Processor.cpp index a760d4ed333977..ceac4f22255a4d 100644 --- a/Kernel/Arch/x86_64/Processor.cpp +++ b/Kernel/Arch/x86_64/Processor.cpp @@ -46,8 +46,7 @@ extern "C" void enter_thread_context(Thread* from_thread, Thread* to_thread) __a extern "C" FlatPtr do_init_context(Thread* thread, u32 flags) __attribute__((used)); extern "C" void syscall_entry(); -template -bool ProcessorBase::is_smp_enabled() +bool ProcessorBase::is_smp_enabled() { return s_smp_enabled; } @@ -58,8 +57,7 @@ UNMAP_AFTER_INIT static void sse_init() write_cr4(read_cr4() | 0x600); } -template -void ProcessorBase::store_fpu_state(FPUState& fpu_state) +void ProcessorBase::store_fpu_state(FPUState& fpu_state) { if (Processor::current().has_feature(CPUFeature::XSAVE) && Processor::current().has_feature(CPUFeature::AVX)) { // The specific state components saved correspond to the bits set in the requested-feature bitmap (RFBM), which is the logical-AND of EDX:EAX and XCR0. @@ -76,8 +74,7 @@ void ProcessorBase::store_fpu_state(FPUState& fpu_state) } } -template -void ProcessorBase::load_fpu_state(FPUState const& fpu_state) +void ProcessorBase::load_fpu_state(FPUState const& fpu_state) { if (Processor::current().has_feature(CPUFeature::XSAVE) && Processor::current().has_feature(CPUFeature::AVX)) asm volatile("xrstor %0" ::"m"(fpu_state), "a"(static_cast(SIMD::StateComponent::AVX | SIMD::StateComponent::SSE | SIMD::StateComponent::X87)), "d"(0u)); @@ -619,8 +616,7 @@ UNMAP_AFTER_INIT void Processor::cpu_setup() m_features |= CPUFeature::OSPKE; } -template -UNMAP_AFTER_INIT void ProcessorBase::early_initialize(u32 cpu) +UNMAP_AFTER_INIT void ProcessorBase::early_initialize(u32 cpu) { m_self = static_cast(this); auto self = static_cast(this); @@ -654,8 +650,7 @@ UNMAP_AFTER_INIT void ProcessorBase::early_initialize(u32 cpu) VERIFY(¤t() == this); // sanity check } -template -UNMAP_AFTER_INIT void ProcessorBase::initialize(u32 cpu) +UNMAP_AFTER_INIT void ProcessorBase::initialize(u32 cpu) { VERIFY(m_self == this); VERIFY(¤t() == this); // sanity check @@ -781,69 +776,12 @@ ProcessorContainer& Processor::processors() return s_processors; } -template -Processor& ProcessorBase::by_id(u32 id) +Processor& ProcessorBase::by_id(u32 id) { return *s_processors[id]; } -template -void ProcessorBase::exit_trap(TrapFrame& trap) -{ - VERIFY_INTERRUPTS_DISABLED(); - VERIFY(&Processor::current() == this); - - auto* self = static_cast(this); - - // Temporarily enter a critical section. This is to prevent critical - // sections entered and left within e.g. smp_process_pending_messages - // to trigger a context switch while we're executing this function - // See the comment at the end of the function why we don't use - // ScopedCritical here. - m_in_critical = m_in_critical + 1; - - VERIFY(m_in_irq >= trap.prev_irq_level); - m_in_irq = trap.prev_irq_level; - - if (s_smp_enabled) - self->smp_process_pending_messages(); - - // Process the deferred call queue. Among other things, this ensures - // that any pending thread unblocks happen before we enter the scheduler. - m_deferred_call_pool.execute_pending(); - - auto* current_thread = Processor::current_thread(); - if (current_thread) { - auto& current_trap = current_thread->current_trap(); - current_trap = trap.next_trap; - ExecutionMode new_previous_mode; - if (current_trap) { - VERIFY(current_trap->regs); - // If we have another higher level trap then we probably returned - // from an interrupt or irq handler. - new_previous_mode = current_trap->regs->previous_mode(); - } else { - // If we don't have a higher level trap then we're back in user mode. - // Which means that the previous mode prior to being back in user mode was kernel mode - new_previous_mode = ExecutionMode::Kernel; - } - - if (current_thread->set_previous_mode(new_previous_mode)) - current_thread->update_time_scheduled(TimeManagement::scheduler_current_time(), true, false); - } - - VERIFY_INTERRUPTS_DISABLED(); - - // Leave the critical section without actually enabling interrupts. - // We don't want context switches to happen until we're explicitly - // triggering a switch in check_invoke_scheduler. - m_in_critical = m_in_critical - 1; - if (!m_in_irq && !m_in_critical) - check_invoke_scheduler(); -} - -template -void ProcessorBase::flush_tlb_local(VirtualAddress vaddr, size_t page_count) +void ProcessorBase::flush_tlb_local(VirtualAddress vaddr, size_t page_count) { auto ptr = vaddr.as_ptr(); while (page_count > 0) { @@ -856,14 +794,12 @@ void ProcessorBase::flush_tlb_local(VirtualAddress vaddr, size_t page_count) } } -template -void ProcessorBase::flush_entire_tlb_local() +void ProcessorBase::flush_entire_tlb_local() { write_cr3(read_cr3()); } -template -void ProcessorBase::flush_tlb(Memory::PageDirectory const* page_directory, VirtualAddress vaddr, size_t page_count) +void ProcessorBase::flush_tlb(Memory::PageDirectory const* page_directory, VirtualAddress vaddr, size_t page_count) { if (s_smp_enabled && (!Memory::is_user_address(vaddr) || Process::current().thread_count() > 1)) Processor::smp_broadcast_flush_tlb(page_directory, vaddr, page_count); @@ -871,8 +807,7 @@ void ProcessorBase::flush_tlb(Memory::PageDirectory const* page_directory, Vi flush_tlb_local(vaddr, page_count); } -template -void ProcessorBase::flush_instruction_cache(VirtualAddress, size_t) +void ProcessorBase::flush_instruction_cache(VirtualAddress, size_t) { // The instruction and data cache are coherent on x86, so we don't need to do anything here. } @@ -916,8 +851,7 @@ ProcessorMessage& Processor::smp_get_from_pool() return *msg; } -template -u32 ProcessorBase::smp_wake_n_idle_processors(u32 wake_count) +u32 ProcessorBase::smp_wake_n_idle_processors(u32 wake_count) { VERIFY_INTERRUPTS_DISABLED(); VERIFY(wake_count > 0); @@ -966,8 +900,7 @@ u32 ProcessorBase::smp_wake_n_idle_processors(u32 wake_count) return did_wake_count; } -template -UNMAP_AFTER_INIT void ProcessorBase::smp_enable() +UNMAP_AFTER_INIT void ProcessorBase::smp_enable() { size_t msg_pool_size = Processor::count() * 100u; size_t msg_entries_cnt = Processor::count(); @@ -1201,8 +1134,7 @@ void Processor::smp_broadcast_halt() APIC::the().broadcast_ipi(); } -template -void ProcessorBase::halt() +void ProcessorBase::halt() { if (s_smp_enabled) Processor::smp_broadcast_halt(); @@ -1298,8 +1230,7 @@ extern "C" NO_SANITIZE_COVERAGE FlatPtr do_init_context(Thread* thread, u32 flag } // FIXME: Share this code with other architectures. -template -void ProcessorBase::assume_context(Thread& thread, InterruptsState new_interrupts_state) +void ProcessorBase::assume_context(Thread& thread, InterruptsState new_interrupts_state) { dbgln_if(CONTEXT_SWITCH_DEBUG, "Assume context for thread {} {}", VirtualAddress(&thread), thread); @@ -1315,8 +1246,7 @@ void ProcessorBase::assume_context(Thread& thread, InterruptsState new_interr VERIFY_NOT_REACHED(); } -template -u32 ProcessorBase::clear_critical() +u32 ProcessorBase::clear_critical() { InterruptDisabler disabler; auto prev_critical = in_critical(); @@ -1365,14 +1295,12 @@ NAKED NO_SANITIZE_COVERAGE void do_assume_context(Thread*, u32) // clang-format on } -template -StringView ProcessorBase::platform_string() +StringView ProcessorBase::platform_string() { return "x86_64"sv; } -template -FlatPtr ProcessorBase::init_context(Thread& thread, bool leave_crit) +FlatPtr ProcessorBase::init_context(Thread& thread, bool leave_crit) { VERIFY(g_scheduler_lock.is_locked()); if (leave_crit) { @@ -1439,7 +1367,6 @@ FlatPtr ProcessorBase::init_context(Thread& thread, bool leave_crit) stack_top -= sizeof(TrapFrame); TrapFrame& trap = *reinterpret_cast(stack_top); trap.regs = &iretframe; - trap.prev_irq_level = 0; trap.next_trap = nullptr; stack_top -= sizeof(u64); // pointer to TrapFrame @@ -1475,8 +1402,7 @@ FlatPtr ProcessorBase::init_context(Thread& thread, bool leave_crit) return stack_top; } -template -void ProcessorBase::switch_context(Thread*& from_thread, Thread*& to_thread) +void ProcessorBase::switch_context(Thread*& from_thread, Thread*& to_thread) { VERIFY(!m_in_irq); VERIFY(m_in_critical == 1); @@ -1558,8 +1484,7 @@ void ProcessorBase::switch_context(Thread*& from_thread, Thread*& to_thread) dbgln_if(CONTEXT_SWITCH_DEBUG, "switch_context <-- from {} {} to {} {}", VirtualAddress(from_thread), *from_thread, VirtualAddress(to_thread), *to_thread); } -template -UNMAP_AFTER_INIT void ProcessorBase::initialize_context_switching(Thread& initial_thread) +UNMAP_AFTER_INIT void ProcessorBase::initialize_context_switching(Thread& initial_thread) { VERIFY(initial_thread.process().is_kernel_process()); auto* self = static_cast(this); @@ -1602,24 +1527,30 @@ void Processor::set_fs_base(FlatPtr fs_base) fs_base_msr.set(fs_base); } -template -void ProcessorBase::idle_begin() const +void ProcessorBase::idle_begin() const { Processor::s_idle_cpu_mask.fetch_or(1u << m_cpu, AK::MemoryOrder::memory_order_relaxed); } -template -void ProcessorBase::idle_end() const +void ProcessorBase::idle_end() const { Processor::s_idle_cpu_mask.fetch_and(~(1u << m_cpu), AK::MemoryOrder::memory_order_relaxed); } -template -void ProcessorBase::wait_for_interrupt() const +void ProcessorBase::idle() const { - asm("hlt"); -} + VERIFY_INTERRUPTS_DISABLED(); + idle_begin(); + asm volatile(R"( + // Atomically go to sleep and enable interrupts. + // These two instructions are atomic, since STI only enables interrupts after the next instruction, so after the processor went to sleep. + sti; hlt + + // Disable interrupts again. + cli + )" :); + idle_end(); } -#include +} diff --git a/Kernel/Arch/x86_64/Processor.h b/Kernel/Arch/x86_64/Processor.h index 2bc59230c91fd1..3c9e8c76f91ef0 100644 --- a/Kernel/Arch/x86_64/Processor.h +++ b/Kernel/Arch/x86_64/Processor.h @@ -44,9 +44,6 @@ struct ProcessorMessageEntry; enum class InterruptsState; class Processor; -template -class ProcessorBase; - // Note: We only support 64 processors at most at the moment, // so allocate 64 slots of inline capacity in the container. @@ -57,10 +54,10 @@ extern "C" void context_first_init(Thread* from_thread, Thread* to_thread, [[may // If this fails to compile because ProcessorBase was not found, you are including this header directly. // Include Arch/Processor.h instead. -class Processor final : public ProcessorBase { +class Processor final : public ProcessorBase { friend class ProcessorInfo; // Allow some implementations to access the idle CPU mask and various x86 implementation details. - friend class ProcessorBase; + friend class ProcessorBase; private: // Saved user stack for the syscall instruction. @@ -158,8 +155,7 @@ class Processor final : public ProcessorBase { static void set_fs_base(FlatPtr); }; -template -ALWAYS_INLINE NO_SANITIZE_COVERAGE Thread* ProcessorBase::current_thread() +ALWAYS_INLINE NO_SANITIZE_COVERAGE Thread* ProcessorBase::current_thread() { // If we were to use ProcessorBase::current here, we'd have to // disable interrupts to prevent a race where we may get pre-empted @@ -167,124 +163,106 @@ ALWAYS_INLINE NO_SANITIZE_COVERAGE Thread* ProcessorBase::current_thread() // to another processor, which would lead us to get the wrong thread. // To avoid having to disable interrupts, we can just read the field // directly in an atomic fashion, similar to Processor::current. - return (Thread*)read_gs_ptr(__builtin_offsetof(ProcessorBase, m_current_thread)); + return (Thread*)read_gs_ptr(__builtin_offsetof(ProcessorBase, m_current_thread)); } -template -ALWAYS_INLINE u32 ProcessorBase::current_id() +ALWAYS_INLINE u32 ProcessorBase::current_id() { // See comment in ProcessorBase::current_thread - return read_gs_ptr(__builtin_offsetof(ProcessorBase, m_cpu)); + return read_gs_ptr(__builtin_offsetof(ProcessorBase, m_cpu)); } -template -ALWAYS_INLINE void ProcessorBase::restore_critical(u32 prev_critical) +ALWAYS_INLINE void ProcessorBase::restore_critical(u32 prev_critical) { // NOTE: This doesn't have to be atomic, and it's also fine if we // get preempted in between these steps. If we move to another // processors m_in_critical will move along with us. And if we // are preempted, we would resume with the same flags. - write_gs_ptr(__builtin_offsetof(ProcessorBase, m_in_critical), prev_critical); + write_gs_ptr(__builtin_offsetof(ProcessorBase, m_in_critical), prev_critical); } -template -ALWAYS_INLINE u32 ProcessorBase::in_critical() +ALWAYS_INLINE u32 ProcessorBase::in_critical() { // See comment in ProcessorBase::current_thread - return read_gs_ptr(__builtin_offsetof(ProcessorBase, m_in_critical)); + return read_gs_ptr(__builtin_offsetof(ProcessorBase, m_in_critical)); } -template -ALWAYS_INLINE void ProcessorBase::set_current_thread(Thread& current_thread) +ALWAYS_INLINE void ProcessorBase::set_current_thread(Thread& current_thread) { // See comment in ProcessorBase::current_thread - write_gs_ptr(__builtin_offsetof(ProcessorBase, m_current_thread), FlatPtr(¤t_thread)); + write_gs_ptr(__builtin_offsetof(ProcessorBase, m_current_thread), FlatPtr(¤t_thread)); } -template -ALWAYS_INLINE Thread* ProcessorBase::idle_thread() +ALWAYS_INLINE Thread* ProcessorBase::idle_thread() { // See comment in ProcessorBase::current_thread - return (Thread*)read_gs_ptr(__builtin_offsetof(ProcessorBase, m_idle_thread)); + return (Thread*)read_gs_ptr(__builtin_offsetof(ProcessorBase, m_idle_thread)); } -template -T& ProcessorBase::current() +Processor& ProcessorBase::current() { - return *(Processor*)(read_gs_ptr(__builtin_offsetof(ProcessorBase, m_self))); + return *(Processor*)(read_gs_ptr(__builtin_offsetof(ProcessorBase, m_self))); } -template -ALWAYS_INLINE bool ProcessorBase::is_initialized() +ALWAYS_INLINE bool ProcessorBase::is_initialized() { - return read_gs_ptr(__builtin_offsetof(ProcessorBase, m_self)) != 0; + return read_gs_ptr(__builtin_offsetof(ProcessorBase, m_self)) != 0; } -template -ALWAYS_INLINE void ProcessorBase::enter_critical() +ALWAYS_INLINE void ProcessorBase::enter_critical() { - write_gs_ptr(__builtin_offsetof(ProcessorBase, m_in_critical), in_critical() + 1); + write_gs_ptr(__builtin_offsetof(ProcessorBase, m_in_critical), in_critical() + 1); } -template -ALWAYS_INLINE NO_SANITIZE_COVERAGE bool ProcessorBase::are_interrupts_enabled() +ALWAYS_INLINE NO_SANITIZE_COVERAGE bool ProcessorBase::are_interrupts_enabled() { return Kernel::are_interrupts_enabled(); } -template -ALWAYS_INLINE bool ProcessorBase::current_in_scheduler() +ALWAYS_INLINE bool ProcessorBase::current_in_scheduler() { - auto value = read_gs_ptr(__builtin_offsetof(ProcessorBase, m_in_scheduler)); + auto value = read_gs_ptr(__builtin_offsetof(ProcessorBase, m_in_scheduler)); return value; } -template -ALWAYS_INLINE void ProcessorBase::set_current_in_scheduler(bool value) +ALWAYS_INLINE void ProcessorBase::set_current_in_scheduler(bool value) { - write_gs_ptr(__builtin_offsetof(ProcessorBase, m_in_scheduler), value); + write_gs_ptr(__builtin_offsetof(ProcessorBase, m_in_scheduler), value); } -template -ALWAYS_INLINE void ProcessorBase::enable_interrupts() +ALWAYS_INLINE void ProcessorBase::enable_interrupts() { sti(); } -template -ALWAYS_INLINE void ProcessorBase::disable_interrupts() +ALWAYS_INLINE void ProcessorBase::disable_interrupts() { cli(); } -template -ALWAYS_INLINE bool ProcessorBase::has_nx() const +ALWAYS_INLINE bool ProcessorBase::has_nx() const { return has_feature(CPUFeature::NX); } -template -ALWAYS_INLINE Optional ProcessorBase::read_cycle_count() +ALWAYS_INLINE Optional ProcessorBase::read_cycle_count() { return read_tsc(); } -template -ALWAYS_INLINE void ProcessorBase::pause() +ALWAYS_INLINE void ProcessorBase::pause() { asm volatile("pause"); } -template -ALWAYS_INLINE void ProcessorBase::wait_check() +ALWAYS_INLINE void ProcessorBase::wait_check() { Processor::pause(); if (Processor::is_smp_enabled()) Processor::current().smp_process_pending_messages(); } -template -ALWAYS_INLINE NO_SANITIZE_COVERAGE FlatPtr ProcessorBase::current_in_irq() +ALWAYS_INLINE NO_SANITIZE_COVERAGE FlatPtr ProcessorBase::current_in_irq() { return read_gs_ptr(__builtin_offsetof(Processor, m_in_irq)); } diff --git a/Kernel/Arch/x86_64/SafeMem.cpp b/Kernel/Arch/x86_64/SafeMem.cpp index fe54941f57c54b..cdf202f7a35954 100644 --- a/Kernel/Arch/x86_64/SafeMem.cpp +++ b/Kernel/Arch/x86_64/SafeMem.cpp @@ -298,7 +298,6 @@ NEVER_INLINE Optional safe_atomic_compare_exchange_relaxed(u32 volatile* v bool handle_safe_access_fault(RegisterState& regs, FlatPtr fault_address) { FlatPtr ip = regs.ip(); - ; if (ip >= (FlatPtr)&start_of_safemem_text && ip < (FlatPtr)&end_of_safemem_text) { // If we detect that the fault happened in safe_memcpy() safe_strnlen(), // or safe_memset() then resume at the appropriate _faulted label diff --git a/Kernel/Arch/x86_64/Time/HPET.cpp b/Kernel/Arch/x86_64/Time/HPET.cpp index 43353eee1d7470..91edf44533b907 100644 --- a/Kernel/Arch/x86_64/Time/HPET.cpp +++ b/Kernel/Arch/x86_64/Time/HPET.cpp @@ -339,7 +339,7 @@ Vector HPET::capable_interrupt_numbers(HPETComparator const& comparato u32 interrupt_bitfield = comparator_registers.interrupt_routing; for (size_t index = 0; index < 32; index++) { if (interrupt_bitfield & 1) - capable_interrupts.append(index); + capable_interrupts.try_append(index).release_value_but_fixme_should_propagate_errors(); interrupt_bitfield >>= 1; } return capable_interrupts; @@ -353,7 +353,7 @@ Vector HPET::capable_interrupt_numbers(u8 comparator_number) u32 interrupt_bitfield = comparator_registers.interrupt_routing; for (size_t index = 0; index < 32; index++) { if (interrupt_bitfield & 1) - capable_interrupts.append(index); + capable_interrupts.try_append(index).release_value_but_fixme_should_propagate_errors(); interrupt_bitfield >>= 1; } return capable_interrupts; @@ -454,8 +454,8 @@ UNMAP_AFTER_INIT HPET::HPET(PhysicalAddress acpi_hpet) if (regs.capabilities.attributes & (u32)HPETFlags::Attributes::LegacyReplacementRouteCapable) regs.configuration.low = regs.configuration.low | (u32)HPETFlags::Configuration::LegacyReplacementRoute; - m_comparators.append(HPETComparator::create(0, 0, is_periodic_capable(0), is_64bit_capable(0))); - m_comparators.append(HPETComparator::create(1, 8, is_periodic_capable(1), is_64bit_capable(1))); + m_comparators.try_append(HPETComparator::create(0, 0, is_periodic_capable(0), is_64bit_capable(0))).release_value_but_fixme_should_propagate_errors(); + m_comparators.try_append(HPETComparator::create(1, 8, is_periodic_capable(1), is_64bit_capable(1))).release_value_but_fixme_should_propagate_errors(); global_enable(); } diff --git a/Kernel/Arch/x86_64/TrapFrame.h b/Kernel/Arch/x86_64/TrapFrame.h deleted file mode 100644 index 719f2e35fbe45f..00000000000000 --- a/Kernel/Arch/x86_64/TrapFrame.h +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Copyright (c) 2018-2021, Andreas Kling - * - * SPDX-License-Identifier: BSD-2-Clause - */ - -#pragma once - -#include - -#include -VALIDATE_IS_X86() - -namespace Kernel { - -struct RegisterState; - -struct TrapFrame { - FlatPtr prev_irq_level; - TrapFrame* next_trap; - RegisterState* regs; // must be last - - TrapFrame() = delete; - TrapFrame(TrapFrame const&) = delete; - TrapFrame(TrapFrame&&) = delete; - TrapFrame& operator=(TrapFrame const&) = delete; - TrapFrame& operator=(TrapFrame&&) = delete; -}; - -#define TRAP_FRAME_SIZE (3 * 8) - -static_assert(AssertSize()); - -} diff --git a/Kernel/Boot/CommandLine.cpp b/Kernel/Boot/CommandLine.cpp index c85c486e874d95..a8d18bbc730711 100644 --- a/Kernel/Boot/CommandLine.cpp +++ b/Kernel/Boot/CommandLine.cpp @@ -36,7 +36,6 @@ UNMAP_AFTER_INIT void CommandLine::initialize() { VERIFY(!s_the); s_the = new CommandLine({ s_cmd_line, strlen(s_cmd_line) }); - dmesgln("Kernel Commandline: {}", kernel_command_line().string()); // Validate the modes the user passed in. (void)s_the->panic_mode(Validate::Yes); } @@ -315,7 +314,7 @@ Vector> CommandLine::userspace_init_args() const if (!init_args.is_empty()) MUST(args.try_prepend(MUST(KString::try_create(userspace_init())))); for (auto& init_arg : init_args) - args.append(MUST(KString::try_create(init_arg))); + MUST(args.try_append(MUST(KString::try_create(init_arg)))); return args; } diff --git a/Kernel/Bus/PCI/Access.cpp b/Kernel/Bus/PCI/Access.cpp index d4f0ae4cc28fb7..ed4a3370af455a 100644 --- a/Kernel/Bus/PCI/Access.cpp +++ b/Kernel/Bus/PCI/Access.cpp @@ -176,7 +176,7 @@ ErrorOr Access::add_host_controller_and_scan_for_devices(NonnullOwnPtr get_bar_bus_address(DeviceIdentifier const& device, HeaderTy auto pci_bar_space_type = get_BAR_space_type(pci_bar_value); if (pci_bar_space_type == BARSpaceType::IOSpace) - return EIO; + return pci_bar_value & bar_io_address_mask; if (pci_bar_space_type == BARSpaceType::Memory64BitSpace) { // FIXME: In theory, BAR5 cannot be assigned to 64 bit as it is the last one... @@ -51,9 +51,6 @@ inline ErrorOr> map_bar(DeviceIdentifier const& device, u64 pci_bar_value = get_BAR(device, bar); auto pci_bar_space_type = get_BAR_space_type(pci_bar_value); - if (pci_bar_space_type == PCI::BARSpaceType::IOSpace) - return EIO; - auto bar_address = TRY(get_bar_address(device, bar)); auto pci_bar_space_size = PCI::get_BAR_space_size(device, bar); diff --git a/Kernel/Bus/PCI/Controller/HostController.cpp b/Kernel/Bus/PCI/Controller/HostController.cpp index bdc49d8bde02b3..60640af279dcae 100644 --- a/Kernel/Bus/PCI/Controller/HostController.cpp +++ b/Kernel/Bus/PCI/Controller/HostController.cpp @@ -24,17 +24,24 @@ ErrorOr HostController::add_memory_space_window(Window const& window) return m_memory_space_windows.try_append(window); } -ErrorOr HostController::translate_bus_address_to_host_address(BARSpaceType space_type, u64 bus_address) const +ErrorOr HostController::add_io_space_window(Window const& window) { - // TODO: Support mapping IO space addresses to memory space addresses. - if (space_type == BARSpaceType::IOSpace) - return ENOTIMPL; + return m_io_space_windows.try_append(window); +} - // If no memory space windows were defined, assume identity mapping. - if (m_memory_space_windows.is_empty()) - return PhysicalAddress { bus_address }; +ErrorOr HostController::translate_bus_address_to_host_address(BARSpaceType space_type, u64 bus_address) const +{ + Vector const* windows = &m_memory_space_windows; + if (space_type == BARSpaceType::IOSpace) { + windows = &m_io_space_windows; + } else { + // If no memory space windows were defined, assume identity mapping. + // For I/O space, this assumption doesn't make sense since we always need to know to which MMIO address they are mapped to. + if (m_memory_space_windows.is_empty()) + return PhysicalAddress { bus_address }; + } - for (auto const& window : m_memory_space_windows) { + for (auto const& window : *windows) { if (bus_address >= window.bus_address && bus_address < (window.bus_address + window.size)) return PhysicalAddress { bus_address - window.bus_address + window.host_address.get() }; } @@ -99,7 +106,7 @@ UNMAP_AFTER_INIT Vector HostController::get_capabilities_for_functio u8 capability_id = capability_header & 0xff; // FIXME: Don't attach a PCI address to a capability object - capabilities.append({ Address(domain_number(), bus.value(), device.value(), function.value()), capability_id, capability_pointer }); + capabilities.try_append({ Address(domain_number(), bus.value(), device.value(), function.value()), capability_id, capability_pointer }).release_value_but_fixme_should_propagate_errors(); capability_pointer = capability_header >> 8; } return capabilities; @@ -145,7 +152,7 @@ UNMAP_AFTER_INIT void HostController::enumerate_functions(Function> 3) & 1; - if (bar_space_type != BARSpaceType::Memory32BitSpace && bar_space_type != BARSpaceType::Memory64BitSpace) - continue; // We only support memory-mapped BAR configuration at the moment + if (bar_space_type == BARSpaceType::Memory32BitSpace) { write32_field(device_identifier.address().bus(), device_identifier.address().device(), device_identifier.address().function(), bar_offset, 0xFFFFFFFF); auto bar_size = read32_field(device_identifier.address().bus(), device_identifier.address().device(), device_identifier.address().function(), bar_offset); write32_field(device_identifier.address().bus(), device_identifier.address().device(), device_identifier.address().function(), bar_offset, bar_value); bar_size &= bar_address_mask; bar_size = (~bar_size) + 1; + if (bar_size == 0) continue; + auto mmio_32bit_address = align_up_to(config.mmio_32bit_base, bar_size); if (mmio_32bit_address + bar_size <= config.mmio_32bit_end) { write32_field(device_identifier.address().bus(), device_identifier.address().device(), device_identifier.address().function(), bar_offset, mmio_32bit_address); config.mmio_32bit_base = mmio_32bit_address + bar_size; continue; } + auto mmio_64bit_address = align_up_to(config.mmio_64bit_base, bar_size); if (bar_prefetchable && mmio_64bit_address + bar_size <= config.mmio_64bit_end && mmio_64bit_address + bar_size <= NumericLimits::max()) { write32_field(device_identifier.address().bus(), device_identifier.address().device(), device_identifier.address().function(), bar_offset, mmio_64bit_address); config.mmio_64bit_base = mmio_64bit_address + bar_size; continue; } + dmesgln("PCI: Ran out of 32-bit MMIO address space"); VERIFY_NOT_REACHED(); + } else if (bar_space_type == BARSpaceType::Memory64BitSpace) { + // 64-bit space + auto next_bar_value = read32_field(device_identifier.address().bus(), device_identifier.address().device(), device_identifier.address().function(), bar_offset + 4); + write32_field(device_identifier.address().bus(), device_identifier.address().device(), device_identifier.address().function(), bar_offset, 0xFFFFFFFF); + write32_field(device_identifier.address().bus(), device_identifier.address().device(), device_identifier.address().function(), bar_offset + 4, 0xFFFFFFFF); + u64 bar_size = read32_field(device_identifier.address().bus(), device_identifier.address().device(), device_identifier.address().function(), bar_offset); + bar_size |= (u64)read32_field(device_identifier.address().bus(), device_identifier.address().device(), device_identifier.address().function(), bar_offset + 4) << 32; + write32_field(device_identifier.address().bus(), device_identifier.address().device(), device_identifier.address().function(), bar_offset, bar_value); + write32_field(device_identifier.address().bus(), device_identifier.address().device(), device_identifier.address().function(), bar_offset + 4, next_bar_value); + bar_size &= bar_address_mask; + bar_size = (~bar_size) + 1; + + if (bar_size == 0) { + bar_offset += 4; + continue; + } + + auto mmio_64bit_address = align_up_to(config.mmio_64bit_base, bar_size); + if (bar_prefetchable && mmio_64bit_address + bar_size <= config.mmio_64bit_end) { + write32_field(device_identifier.address().bus(), device_identifier.address().device(), device_identifier.address().function(), bar_offset, mmio_64bit_address & 0xFFFFFFFF); + write32_field(device_identifier.address().bus(), device_identifier.address().device(), device_identifier.address().function(), bar_offset + 4, mmio_64bit_address >> 32); + config.mmio_64bit_base = mmio_64bit_address + bar_size; + bar_offset += 4; + continue; + } + + auto mmio_32bit_address = align_up_to(config.mmio_32bit_base, bar_size); + if (mmio_32bit_address + bar_size <= config.mmio_32bit_end) { + write32_field(device_identifier.address().bus(), device_identifier.address().device(), device_identifier.address().function(), bar_offset, mmio_32bit_address & 0xFFFFFFFF); + write32_field(device_identifier.address().bus(), device_identifier.address().device(), device_identifier.address().function(), bar_offset + 4, mmio_32bit_address >> 32); + config.mmio_32bit_base = mmio_32bit_address + bar_size; + bar_offset += 4; + continue; + } + + dmesgln("PCI: Ran out of 64-bit MMIO address space"); + VERIFY_NOT_REACHED(); + } else if (bar_space_type == BARSpaceType::IOSpace) { + write32_field(device_identifier.address().bus(), device_identifier.address().device(), device_identifier.address().function(), bar_offset, 0xFFFFFFFF); + auto bar_size = read32_field(device_identifier.address().bus(), device_identifier.address().device(), device_identifier.address().function(), bar_offset); + write32_field(device_identifier.address().bus(), device_identifier.address().device(), device_identifier.address().function(), bar_offset, bar_value); + bar_size &= bar_io_address_mask; + bar_size = (~bar_size) + 1; + if (bar_size == 0) + continue; + + auto io_address = align_up_to(config.io_base, bar_size); + if (io_address + bar_size <= config.io_end) { + write32_field(device_identifier.address().bus(), device_identifier.address().device(), device_identifier.address().function(), bar_offset, io_address); + config.io_base = io_address + bar_size; + continue; + } + + dmesgln("PCI: Ran out of I/O address space"); + VERIFY_NOT_REACHED(); } - // 64-bit space - auto next_bar_value = read32_field(device_identifier.address().bus(), device_identifier.address().device(), device_identifier.address().function(), bar_offset + 4); - write32_field(device_identifier.address().bus(), device_identifier.address().device(), device_identifier.address().function(), bar_offset, 0xFFFFFFFF); - write32_field(device_identifier.address().bus(), device_identifier.address().device(), device_identifier.address().function(), bar_offset + 4, 0xFFFFFFFF); - u64 bar_size = read32_field(device_identifier.address().bus(), device_identifier.address().device(), device_identifier.address().function(), bar_offset); - bar_size |= (u64)read32_field(device_identifier.address().bus(), device_identifier.address().device(), device_identifier.address().function(), bar_offset + 4) << 32; - write32_field(device_identifier.address().bus(), device_identifier.address().device(), device_identifier.address().function(), bar_offset, bar_value); - write32_field(device_identifier.address().bus(), device_identifier.address().device(), device_identifier.address().function(), bar_offset + 4, next_bar_value); - bar_size &= bar_address_mask; - bar_size = (~bar_size) + 1; - if (bar_size == 0) { - bar_offset += 4; - continue; - } - auto mmio_64bit_address = align_up_to(config.mmio_64bit_base, bar_size); - if (bar_prefetchable && mmio_64bit_address + bar_size <= config.mmio_64bit_end) { - write32_field(device_identifier.address().bus(), device_identifier.address().device(), device_identifier.address().function(), bar_offset, mmio_64bit_address & 0xFFFFFFFF); - write32_field(device_identifier.address().bus(), device_identifier.address().device(), device_identifier.address().function(), bar_offset + 4, mmio_64bit_address >> 32); - config.mmio_64bit_base = mmio_64bit_address + bar_size; - bar_offset += 4; - continue; - } - auto mmio_32bit_address = align_up_to(config.mmio_32bit_base, bar_size); - if (mmio_32bit_address + bar_size <= config.mmio_32bit_end) { - write32_field(device_identifier.address().bus(), device_identifier.address().device(), device_identifier.address().function(), bar_offset, mmio_32bit_address & 0xFFFFFFFF); - write32_field(device_identifier.address().bus(), device_identifier.address().device(), device_identifier.address().function(), bar_offset + 4, mmio_32bit_address >> 32); - config.mmio_32bit_base = mmio_32bit_address + bar_size; - bar_offset += 4; - continue; - } - dmesgln("PCI: Ran out of 64-bit MMIO address space"); - VERIFY_NOT_REACHED(); } + // enable memory space auto command_value = read16_field(device_identifier.address().bus(), device_identifier.address().device(), device_identifier.address().function(), PCI::RegisterOffset::COMMAND); command_value |= 1 << 1; // memory space enable write16_field(device_identifier.address().bus(), device_identifier.address().device(), device_identifier.address().function(), PCI::RegisterOffset::COMMAND, command_value); + // assign interrupt number auto interrupt_pin = read8_field(device_identifier.address().bus(), device_identifier.address().device(), device_identifier.address().function(), PCI::RegisterOffset::INTERRUPT_PIN); - auto masked_identifier = PCIInterruptSpecifier{ + auto masked_identifier = PCIInterruptSpecifier { .interrupt_pin = interrupt_pin, .function = device_identifier.address().function(), .device = device_identifier.address().device(), .bus = device_identifier.address().bus() }; masked_identifier &= config.interrupt_mask; + auto interrupt_number = config.masked_interrupt_mapping.get(masked_identifier); if (interrupt_number.has_value()) write8_field(device_identifier.address().bus(), device_identifier.address().device(), device_identifier.address().function(), PCI::RegisterOffset::INTERRUPT_LINE, interrupt_number.value()); @@ -319,19 +356,31 @@ void HostController::configure_attached_devices(PCIConfiguration& config) return; if (read8_field(device_identifier.address().bus(), device_identifier.address().device(), device_identifier.address().function(), PCI::RegisterOffset::SUBCLASS) != PCI::Bridge::SubclassID::PCI_TO_PCI) return; + // bridge-specific handling + config.io_base = align_up_to(config.io_base, 4 * KiB); + write8_field(device_identifier.address().bus(), device_identifier.address().device(), device_identifier.address().function(), PCI::RegisterOffset::IO_BASE, config.io_base >> 8); + write16_field(device_identifier.address().bus(), device_identifier.address().device(), device_identifier.address().function(), PCI::RegisterOffset::IO_BASE_UPPER_16_BITS, config.io_base >> 16); + config.mmio_32bit_base = align_up_to(config.mmio_32bit_base, MiB); - config.mmio_64bit_base = align_up_to(config.mmio_64bit_base, MiB); write16_field(device_identifier.address().bus(), device_identifier.address().device(), device_identifier.address().function(), PCI::RegisterOffset::MEMORY_BASE, config.mmio_32bit_base >> 16); + + config.mmio_64bit_base = align_up_to(config.mmio_64bit_base, MiB); write16_field(device_identifier.address().bus(), device_identifier.address().device(), device_identifier.address().function(), PCI::RegisterOffset::PREFETCHABLE_MEMORY_BASE, config.mmio_64bit_base >> 16); write32_field(device_identifier.address().bus(), device_identifier.address().device(), device_identifier.address().function(), PCI::RegisterOffset::PREFETCHABLE_MEMORY_BASE_UPPER_32_BITS, config.mmio_64bit_base >> 32); }, + [this, &config](EnumerableDeviceIdentifier const& device_identifier) { // called after a bridge was recursively enumerated + write8_field(device_identifier.address().bus(), device_identifier.address().device(), device_identifier.address().function(), PCI::RegisterOffset::IO_LIMIT, config.io_base >> 8); + write16_field(device_identifier.address().bus(), device_identifier.address().device(), device_identifier.address().function(), PCI::RegisterOffset::IO_LIMIT_UPPER_16_BITS, config.io_base >> 16); + config.mmio_32bit_base = align_up_to(config.mmio_32bit_base, MiB); - config.mmio_64bit_base = align_up_to(config.mmio_64bit_base, MiB); write16_field(device_identifier.address().bus(), device_identifier.address().device(), device_identifier.address().function(), PCI::RegisterOffset::MEMORY_LIMIT, config.mmio_32bit_base >> 16); + + config.mmio_64bit_base = align_up_to(config.mmio_64bit_base, MiB); write16_field(device_identifier.address().bus(), device_identifier.address().device(), device_identifier.address().function(), PCI::RegisterOffset::PREFETCHABLE_MEMORY_LIMIT, config.mmio_64bit_base >> 16); write32_field(device_identifier.address().bus(), device_identifier.address().device(), device_identifier.address().function(), PCI::RegisterOffset::PREFETCHABLE_MEMORY_LIMIT_UPPER_32_BITS, config.mmio_64bit_base >> 32); + // enable bridging auto command_value = read16_field(device_identifier.address().bus(), device_identifier.address().device(), device_identifier.address().function(), PCI::RegisterOffset::COMMAND); command_value |= 1 << 2; // enable forwarding of requests by the bridge diff --git a/Kernel/Bus/PCI/Controller/HostController.h b/Kernel/Bus/PCI/Controller/HostController.h index 37eb95e985869e..18ca6baef516e3 100644 --- a/Kernel/Bus/PCI/Controller/HostController.h +++ b/Kernel/Bus/PCI/Controller/HostController.h @@ -61,8 +61,13 @@ namespace Kernel::PCI { struct PCIConfiguration { FlatPtr mmio_32bit_base { 0 }; FlatPtr mmio_32bit_end { 0 }; + FlatPtr mmio_64bit_base { 0 }; FlatPtr mmio_64bit_end { 0 }; + + FlatPtr io_base { 0 }; + FlatPtr io_end { 0 }; + // The keys contains the bus, device & function at the same offsets as OpenFirmware PCI addresses, // with the least significant 8 bits being the interrupt pin. HashMap masked_interrupt_mapping; @@ -95,6 +100,7 @@ class HostController { }; ErrorOr add_memory_space_window(Window const&); + ErrorOr add_io_space_window(Window const&); private: void enumerate_bus(Function const& callback, Function& post_bridge_callback, BusNumber, bool recursive_search_into_bridges); @@ -130,6 +136,7 @@ class HostController { Bitmap m_enumerated_buses; Vector m_memory_space_windows; + Vector m_io_space_windows; }; } diff --git a/Kernel/Bus/PCI/Definitions.h b/Kernel/Bus/PCI/Definitions.h index 7be87ea0a6eabf..9a1977a91b8281 100644 --- a/Kernel/Bus/PCI/Definitions.h +++ b/Kernel/Bus/PCI/Definitions.h @@ -60,6 +60,8 @@ enum class RegisterOffset : u32 { SECONDARY_BUS = 0x19, // byte SUBORDINATE_BUS = 0x1A, // byte BAR3 = 0x1C, // u32 + IO_BASE = 0x1c, // byte + IO_LIMIT = 0x1d, // byte BAR4 = 0x20, // u32 MEMORY_BASE = 0x20, // u16 MEMORY_LIMIT = 0x22, // u16 @@ -71,6 +73,8 @@ enum class RegisterOffset : u32 { SUBSYSTEM_VENDOR_ID = 0x2C, // u16 SUBSYSTEM_ID = 0x2E, // u16 EXPANSION_ROM_POINTER = 0x30, // u32 + IO_BASE_UPPER_16_BITS = 0x30, // u16 + IO_LIMIT_UPPER_16_BITS = 0x32, // u16 CAPABILITIES_POINTER = 0x34, // u8 INTERRUPT_LINE = 0x3C, // byte INTERRUPT_PIN = 0x3D, // byte @@ -89,6 +93,7 @@ static constexpr size_t mmio_device_space_size = 4096; static constexpr u16 none_value = 0xffff; static constexpr size_t memory_range_per_bus = mmio_device_space_size * to_underlying(Limits::MaxFunctionsPerDevice) * to_underlying(Limits::MaxDevicesPerBus); static constexpr u64 bar_address_mask = ~0xfull; +static constexpr u64 bar_io_address_mask = ~0x3ull; static constexpr u8 msi_control_offset = 2; static constexpr u16 msi_control_enable = 0x0001; static constexpr u8 msi_address_low_offset = 4; @@ -443,7 +448,7 @@ AK_TYPEDEF_DISTINCT_ORDERED_ID(u8, InterruptPin); class Access; class EnumerableDeviceIdentifier { public: - EnumerableDeviceIdentifier(Address address, HardwareID hardware_id, RevisionID revision_id, ClassCode class_code, SubclassCode subclass_code, ProgrammingInterface prog_if, SubsystemID subsystem_id, SubsystemVendorID subsystem_vendor_id, InterruptLine interrupt_line, InterruptPin interrupt_pin, Vector const& capabilities) + EnumerableDeviceIdentifier(Address address, HardwareID hardware_id, RevisionID revision_id, ClassCode class_code, SubclassCode subclass_code, ProgrammingInterface prog_if, SubsystemID subsystem_id, SubsystemVendorID subsystem_vendor_id, InterruptLine interrupt_line, InterruptPin interrupt_pin, Vector capabilities) : m_address(address) , m_hardware_id(hardware_id) , m_revision_id(revision_id) @@ -454,7 +459,7 @@ class EnumerableDeviceIdentifier { , m_subsystem_vendor_id(subsystem_vendor_id) , m_interrupt_line(interrupt_line) , m_interrupt_pin(interrupt_pin) - , m_capabilities(capabilities) + , m_capabilities(move(capabilities)) { if constexpr (PCI_DEBUG) { for (auto const& capability : capabilities) @@ -569,7 +574,7 @@ class DeviceIdentifier other_identifier.subsystem_vendor_id(), other_identifier.interrupt_line(), other_identifier.interrupt_pin(), - other_identifier.capabilities()) + other_identifier.capabilities().clone().release_value_but_fixme_should_propagate_errors()) { } diff --git a/Kernel/Bus/PCI/DeviceTreeHelpers.cpp b/Kernel/Bus/PCI/DeviceTreeHelpers.cpp index c9512848bcd9e8..39011dcca895fa 100644 --- a/Kernel/Bus/PCI/DeviceTreeHelpers.cpp +++ b/Kernel/Bus/PCI/DeviceTreeHelpers.cpp @@ -7,6 +7,7 @@ #include #include #include +#include #include #include @@ -79,6 +80,8 @@ ErrorOr configure_devicetree_host_controller(HostController& host_controll u32 pci_32bit_mmio_size = 0; FlatPtr pci_64bit_mmio_base = 0; u64 pci_64bit_mmio_size = 0; + FlatPtr pci_io_base = 0; + u64 pci_io_size = 0; HashMap masked_interrupt_mapping; PCIInterruptSpecifier interrupt_mask; @@ -114,17 +117,13 @@ ErrorOr configure_devicetree_host_controller(HostController& host_controll !!pci_address.prefetchable, !pci_address.non_relocatable); - if (pci_address.space_type != OpenFirmwareAddress::SpaceType::Memory32BitSpace - && pci_address.space_type != OpenFirmwareAddress::SpaceType::Memory64BitSpace) - continue; // We currently only support memory-mapped PCI on RISC-V and AArch64 - - TRY(host_controller.add_memory_space_window(HostController::Window { - .host_address = PhysicalAddress { cpu_physical_address }, - .bus_address = pci_address.io_or_memory_space_address, - .size = range_size, - })); - if (pci_address.space_type == OpenFirmwareAddress::SpaceType::Memory32BitSpace) { + TRY(host_controller.add_memory_space_window(HostController::Window { + .host_address = PhysicalAddress { cpu_physical_address }, + .bus_address = pci_address.io_or_memory_space_address, + .size = range_size, + })); + if (pci_address.prefetchable) continue; // We currently only use non-prefetchable 32-bit regions, since 64-bit regions are always prefetchable - TODO: Use 32-bit prefetchable regions if only they are available if (pci_32bit_mmio_size >= range_size) @@ -132,12 +131,32 @@ ErrorOr configure_devicetree_host_controller(HostController& host_controll pci_32bit_mmio_base = pci_address.io_or_memory_space_address; pci_32bit_mmio_size = range_size; - } else { + } else if (pci_address.space_type == OpenFirmwareAddress::SpaceType::Memory64BitSpace) { + TRY(host_controller.add_memory_space_window(HostController::Window { + .host_address = PhysicalAddress { cpu_physical_address }, + .bus_address = pci_address.io_or_memory_space_address, + .size = range_size, + })); + if (pci_64bit_mmio_size >= range_size) continue; // We currently only use the single largest region - TODO: Use all available regions if needed pci_64bit_mmio_base = pci_address.io_or_memory_space_address; pci_64bit_mmio_size = range_size; + } else if (pci_address.space_type == OpenFirmwareAddress::SpaceType::IOSpace) { + TRY(host_controller.add_io_space_window(HostController::Window { + .host_address = PhysicalAddress { cpu_physical_address }, + .bus_address = pci_address.io_or_memory_space_address, + .size = range_size, + })); + + if (pci_io_size >= range_size) + continue; // We currently only use the single largest region - TODO: Use all available regions if needed + + pci_io_base = pci_address.io_or_memory_space_address; + pci_io_size = range_size; + } else { + dbgln("PCI: Unexpected address space type in 'ranges' property: {}", to_underlying(pci_address.space_type)); } } } @@ -208,11 +227,6 @@ ErrorOr configure_devicetree_host_controller(HostController& host_controll if (interrupt_controller == nullptr) return EINVAL; - if (!interrupt_controller->has_property("interrupt-controller"sv)) { - dmesgln("PCI: Implement support for nested interrupt nexuses"); - TODO(); - } - // The devicetree spec says that we should assume `#address-cells = <2>` if the property isn't present. // Linux however defaults to `#address-cells = <0>` for interrupt parent nodes. // Some devicetrees seem to expect this Linux behavior, so we have to follow it. @@ -224,26 +238,16 @@ ErrorOr configure_devicetree_host_controller(HostController& host_controll TRY(map_stream.discard(sizeof(u32) * interrupt_controller_address_cells)); auto interrupt_cells = interrupt_controller->get_property("#interrupt-cells"sv)->as(); -#if ARCH(RISCV64) - if (interrupt_cells != 1 && interrupt_cells != 2) - return ENOTSUP; - u64 interrupt = TRY(map_stream.read_cells(interrupt_cells)); -#elif ARCH(AARCH64) - // FIXME: Don't depend on a specific interrupt descriptor format. - auto const& domain_root = *TRY(interrupt_controller->interrupt_domain_root(device_tree)); - if (!domain_root.is_compatible_with("arm,gic-400"sv) && !domain_root.is_compatible_with("arm,cortex-a15-gic"sv)) - return ENOTSUP; + auto interrupt_specifier = TRY(map_stream.read_in_place(interrupt_cells * sizeof(u32))); + auto const* domain_root = TRY(interrupt_controller->interrupt_domain_root(device_tree)); - if (interrupt_cells != 3) + if (!domain_root->has_property("interrupt-controller"sv)) { + dmesgln("PCI: Implement support for nested interrupt nexuses"); return ENOTSUP; + } - TRY(map_stream.discard(sizeof(u32))); // This is the IRQ type. - u64 interrupt = TRY(map_stream.read_cell()) + 32; - TRY(map_stream.discard(sizeof(u32))); // This is the trigger type. -#else -# error Unknown architecture -#endif + auto interrupt_number = TRY(DeviceTree::Management::the().resolve_interrupt_number({ domain_root, interrupt_specifier })); pin &= pin_mask; pci_address &= pci_address_mask; @@ -254,7 +258,7 @@ ErrorOr configure_devicetree_host_controller(HostController& host_controll .device = pci_address.device, .bus = pci_address.bus, }, - interrupt); + interrupt_number); } } @@ -264,6 +268,8 @@ ErrorOr configure_devicetree_host_controller(HostController& host_controll pci_32bit_mmio_base + pci_32bit_mmio_size, pci_64bit_mmio_base, pci_64bit_mmio_base + pci_64bit_mmio_size, + pci_io_base, + pci_io_base + pci_io_size, move(masked_interrupt_mapping), interrupt_mask, }; diff --git a/Kernel/Bus/USB/Drivers/CDC/Codes.h b/Kernel/Bus/USB/Drivers/CDC/Codes.h new file mode 100644 index 00000000000000..6a766fa664e614 --- /dev/null +++ b/Kernel/Bus/USB/Drivers/CDC/Codes.h @@ -0,0 +1,194 @@ +/* + * Copyright (c) 2025, Leon Albrecht + * + * SPDX-License-Identifier: BSD-2-Clause + */ + +#pragma once + +#include +#include +#include +#include +#include +#include + +namespace Kernel::USB::CDC { + +// https://www.usb.org/sites/default/files/CDC1.2_WMC1.1_012011_0.zip +#define ENUMERATE_SUBCLASS_CODES(X) \ + X(0x00, Reserved, "Reserved", "[None]") \ + X(0x01, DirectLineControlModel, "Direct Line Control Model", "[USBPSTN]") \ + X(0x02, AbstractControlModel, "Abstract Control Model", "[USBPSTN]") \ + X(0x03, TelephoneControlModel, "Telephone Control Model", "[USBPSTN]") \ + X(0x04, MultiChannelControlModel, "Multi-Channel Control Model", "[USBISDN]") \ + X(0x05, CAPIControlModel, "CAPI Control Model", "[USBISDN]") \ + X(0x06, EthernetNetworkingControlModel, "Ethernet Networking Control Model", "[USBECM]") \ + X(0x07, ATMNetworkingControlModel, "ATM Networking Control Model", "[USBATM]") \ + X(0x08, WirelessHandsetControlModel, "Wireless Handset Control Model", "[USBWMC]") \ + X(0x09, DeviceManagement, "Device Management", "[USBWMC]") \ + X(0x0A, MobileDirectLineModel, "Mobile Direct Line Model", "[USBWMC]") \ + X(0x0B, OBEX, "OBEX", "[USBWMC]") \ + X(0x0C, EthernetEmulationModel, "Ethernet Emulation Model", "[USBEEM]") \ + X(0x0D, NetworkControlModel, "Network Control Model", "[USBNCM]") \ + X(0x0E, MobileBroadbandInterfaceModel, "Mobile Broadband Interface Model", "[USBMBIM]") +// 0x0F-0x7F Reserved +// 0x80-0xFF Vendor Specific + +enum class SubclassCode : u8 { +#define __ENUMERATE_SUBCLASS_CODE(code, name, _, __) name = code, + ENUMERATE_SUBCLASS_CODES(__ENUMERATE_SUBCLASS_CODE) +#undef __ENUMERATE_SUBCLASS_CODE +}; + +inline StringView subclass_code_to_string(SubclassCode code) +{ + switch (to_underlying(code)) { +#define __ENUMERATE_SUBCLASS_CODE(code, name, str, standard) \ + case to_underlying(SubclassCode::name): \ + return standard " " str##sv; + ENUMERATE_SUBCLASS_CODES(__ENUMERATE_SUBCLASS_CODE) + case 0x0F ... 0x7F: + return "Reserved"sv; + case 0x80 ... 0xFF: + return "Vendor Specific"sv; + default: + VERIFY_NOT_REACHED(); + } +#undef __ENUMERATE_SUBCLASS_CODE +} + +#define ENUMERATE_COMMUNICATION_PROTOCOL_CODES(X) \ + X(0x00, NoSpecificProtocol, "No Specific Protocol", "[USB]") \ + X(0x01, ATCommands_V250, "AT Commands", "ITU-T V.250") \ + X(0x02, ATCommands_PCCA101, "AT Commands", "PCCA-101") \ + X(0x03, ATCommands_PCCA101_AnnexO, "AT Commands", "PCCA-101 Annex O") \ + X(0x04, ATCommands_GSM07_07, "AT Commands", "GSM 7.07") \ + X(0x05, ATCommands_3GPP27_007, "AT Commands", "3GPP 27.007") \ + X(0x06, ATCommands_TIA_CDMA, "AT Commands (TIA CDMA)", "C-S0017-0") \ + X(0x07, EthernetEmulationModel, "Ethernet Emulation Model", "[USBEEM]") \ + /* 08h-FDh Reserved (future use) */ \ + X(0xFE, ExternalProtocol, "External Protocol", "") \ + X(0xFF, Vendor, "Vendor-Specific", "[USB]") + +enum class CommunicationProtocolCode : u8 { +#define __ENUMERATE_PROTOCOL_CODE(code, name, _, __) name = code, + ENUMERATE_COMMUNICATION_PROTOCOL_CODES(__ENUMERATE_PROTOCOL_CODE) +#undef __ENUMERATE_PROTOCOL_CODE +}; + +inline StringView protocol_code_to_string(CommunicationProtocolCode code) +{ + switch (to_underlying(code)) { +#define __ENUMERATE_PROTOCOL_CODE(code, name, str, standard) \ + case to_underlying(CommunicationProtocolCode::name): \ + return standard " " str##sv; + ENUMERATE_COMMUNICATION_PROTOCOL_CODES(__ENUMERATE_PROTOCOL_CODE) +#undef __ENUMERATE_PROTOCOL_CODE + default: + return "Reserved"sv; + } +} + +#define ENUMERATE_DATA_PROTOCOL_CODES(X) \ + X(0x00, NoSpecificProtocol, "No Specific Protocol", "[USB]") \ + X(0x01, NetworkTransferBlock, "Network Transfer Block", "[USBNCM]") \ + /* 02h-2Fh Reserved (future use) */ \ + X(0x30, I430, "Physical interface protocol for ISDN BRI", "I.430") \ + X(0x31, ISO3309, "HDLC", "[ISO/IEC 3309-1993]") \ + X(0x32, Transparent, "Transparent", "[None]") \ + /* 33h-4Fh Reserved (future use) */ \ + X(0x50, Q921M, "Management protocol for Q.921 data link protocol", "Q.921M") \ + X(0x51, Q921, "Data link protocol for Q.931", "Q.921") \ + X(0x52, Q921TM, "TEI-multiplexor for Q.921 data link protocol", "Q.921TM") \ + /* 53h-8Fh Reserved (future use) */ \ + X(0x90, V42bis, "V.42bis", "[Data compression procedures]") \ + X(0x91, Q931EuroISDN, "Euro-ISDN protocol control", "Q.931/Euro- ISDN") \ + X(0x92, V120, "V.24 rate adaptation to ISDN", "V120") \ + X(0x93, CAPI20, "CAPI Commands", "CAPI2.0") \ + /* 0x94 - 0xFC Reserved (future use) */ \ + X(0xFD, HostBasedDriver, "Host based driver", "[None]") \ + /* Note: This protocol code should only be used in messages between host and device to identify the host driver portion of a protocol stack. */ \ + X(0xFE, FunctionalDescriptor, "Protocol Unit Functional Descriptor Defined", "[USBCDC]") \ + /* The protocol(s) are described using a Protocol Unit Functional Descriptors on Communications Class Interface */ \ + X(0xFF, Vendor, "Vendor-Specific", "[USB]") + +enum class DataProtocolCode : u8 { +#define __ENUMERATE_PROTOCOL_CODE(code, name, _, __) name = code, + ENUMERATE_DATA_PROTOCOL_CODES(__ENUMERATE_PROTOCOL_CODE) +#undef __ENUMERATE_PROTOCOL_CODE +}; + +inline StringView protocol_code_to_string(DataProtocolCode code) +{ + switch (to_underlying(code)) { +#define __ENUMERATE_PROTOCOL_CODE(code, name, str, standard) \ + case to_underlying(DataProtocolCode::name): \ + return standard " " str##sv; + ENUMERATE_DATA_PROTOCOL_CODES(__ENUMERATE_PROTOCOL_CODE) +#undef __ENUMERATE_PROTOCOL_CODE + default: + return "Reserved"sv; + } +} + +enum class ClassSpecificDescriptorCodes : u8 { + CS_Interface = 0x24, + CS_Endpoint = 0x25 +}; + +#define ENUMERATE_CDC_CS_INTERFACE_DESCRIPTORS(X) \ + X(0x00, Header, "Header") \ + X(0x01, CallManagement, "Call Management") \ + X(0x02, AbstractControlManagement, "Abstract Control Management") \ + X(0x03, DirectLineManagement, "Direct Line Management") \ + X(0x04, TelephoneRingerManagement, "Telephone Ringer Management") \ + X(0x05, TelephoneCallAndLineStateReportingCapabilitiesDescriptor, "Telephone Call and Line State Reporting Capabilities Descriptor") \ + X(0x06, Union, "Union") \ + X(0x07, CountrySelection, "Country Selection") \ + X(0x08, TelephoneOperationalModes, "Telephone Operational Modes") \ + X(0x09, USBTerminal, "USB Terminal") \ + X(0x0A, NetworkChannelTerminal, "Network Channel Terminal") \ + X(0x0B, ProtocolUnit, "Protocol Unit") \ + X(0x0C, ExtensionUnit, "Extension Unit") \ + X(0x0D, MultiChannelManagement, "Multi-Channel Management") \ + X(0x0E, CAPIControlManagement, "CAPI Control Management") \ + X(0x0F, EthernetNetworking, "Ethernet Networking") \ + X(0x10, ATMNetworking, "ATM Networking") \ + X(0x11, WirelessHandsetControlModel, "Wireless Handset Control Model") \ + X(0x12, MobileDirectLineModel, "Mobile Direct Line Model") \ + X(0x13, MDLMDetail, "MDLM Detail") \ + X(0x14, DeviceManagementModel, "Device Management Model") \ + X(0x15, OBEX, "OBEX") \ + X(0x16, CommandSet, "Command Set") \ + X(0x17, CommandSetDetail, "Command Set Detail") \ + X(0x18, TelephoneControlModel, "Telephone Control Model") \ + X(0x19, OBEXServiceIdentifier, "OBEX Service Identifier") \ + X(0x1A, NCM, "NCM") \ + X(0x1B, MBIM, "MBIM") \ + X(0x1C, ExtendedNBIM, "Extended NBIM") \ + X(0x1D, NCMExtendedCapability, "NCM Extended Capabilities") \ + X(0x1E, NCMExtendedFeature, "NCM Extended Feature") \ + /*0x1F-0x7F: Reserved (future use)*/ \ + /*0x80-0xFF: Vendor-specific*/ + +enum class ClassSpecificInterfaceDescriptorCodes : u8 { +#define __ENUMERATE_CS_INTERFACE_DESCRIPTOR(code, name, _) name = code, + ENUMERATE_CDC_CS_INTERFACE_DESCRIPTORS(__ENUMERATE_CS_INTERFACE_DESCRIPTOR) +#undef __ENUMERATE_CS_INTERFACE_DESCRIPTOR +}; + +inline StringView class_specific_interface_descriptor_to_string(ClassSpecificInterfaceDescriptorCodes code) +{ + switch (to_underlying(code)) { +#define __ENUMERATE_CS_INTERFACE_DESCRIPTOR(code, name, str) \ + case to_underlying(ClassSpecificInterfaceDescriptorCodes::name): \ + return str##sv; + ENUMERATE_CDC_CS_INTERFACE_DESCRIPTORS(__ENUMERATE_CS_INTERFACE_DESCRIPTOR) +#undef __ENUMERATE_CS_INTERFACE_DESCRIPTOR + default: + return "Reserved"sv; + } +} + +} diff --git a/Kernel/Bus/USB/Drivers/CDC/Driver.cpp b/Kernel/Bus/USB/Drivers/CDC/Driver.cpp new file mode 100644 index 00000000000000..32ff4e11dc3d11 --- /dev/null +++ b/Kernel/Bus/USB/Drivers/CDC/Driver.cpp @@ -0,0 +1,261 @@ +/* + * Copyright (c) 2025, Leon Albrecht + * + * SPDX-License-Identifier: BSD-2-Clause + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace Kernel::USB::CDC { + +class CDCDriver final : public Driver { +public: + CDCDriver() + : Driver("USB CDC"sv) + { + } + + static void init(); + + virtual ~CDCDriver() override = default; + + virtual ErrorOr probe(USB::Device&) override; + virtual void detach(USB::Device&) override { } +}; + +USB_DEVICE_DRIVER(CDCDriver); + +void CDCDriver::init() +{ + auto driver = MUST(adopt_nonnull_lock_ref_or_enomem(new (nothrow) CDCDriver())); + MUST(USBManagement::register_driver(driver)); +} + +static ErrorOr dump_cdc_data_interface(USB::USBConfiguration const&, USB::USBInterface const& interface) +{ + auto protocol_code = interface.descriptor().interface_protocol; + + dmesgln("USB CDC: Data Interface, protocol: ({:#02X}) {}", protocol_code, protocol_code_to_string(CDC::DataProtocolCode { protocol_code })); + dmesgln("USB CDC: Endpoints: {}", interface.endpoints().size()); + + return {}; +} + +static ErrorOr dump_cdc_functional_descriptor(USB::USBConfiguration const& configuration, USB::USBInterface const&, ReadonlyBytes raw_descriptor) +{ + auto& device = const_cast(configuration.device()); // FIXME: Evil!! + + auto const& descriptor_header = *reinterpret_cast(raw_descriptor.data()); + VERIFY(descriptor_header.descriptor_type == static_cast(CDC::ClassSpecificDescriptorCodes::CS_Interface)); + + auto subtype = raw_descriptor[2]; + dmesgln("USB CDC: ({:#02X}) {}, length: {}", subtype, class_specific_interface_descriptor_to_string(static_cast(subtype)), descriptor_header.length); + + switch (static_cast(subtype)) { + case CDC::ClassSpecificInterfaceDescriptorCodes::Union: { + // Union Functional Descriptor + // After the header and subtype, the first byte is the master/controlling interface, + // followed by a list subordinate interfaces. + auto master_interface = raw_descriptor[3]; + dmesgln("USB CDC: Control Interface: {}", master_interface); + dmesgln("USB CDC: Subordinates: {}", raw_descriptor.slice(sizeof(USBDescriptorCommon) + 2)); + } break; + case CDC::ClassSpecificInterfaceDescriptorCodes::EthernetNetworking: { + auto stream = FixedMemoryStream(raw_descriptor.slice(sizeof(USBDescriptorCommon) + 1)); + u8 i_macaddr = TRY(stream.read_value()); + u32 bm_stats = TRY(stream.read_value>()); + u16 w_max_segment_size = TRY(stream.read_value>()); + u16 w_number_mc_filters = TRY(stream.read_value>()); + u8 b_number_power_filters = TRY(stream.read_value()); + dmesgln("USB CDC: MAC Address String Index: {}", i_macaddr); + auto maybe_mac_address = device.get_string_descriptor(i_macaddr); + if (maybe_mac_address.is_error()) { + dmesgln("USB CDC: Failed to get MAC address string descriptor"); + } else { + dmesgln("USB CDC: MAC Address: {}", maybe_mac_address.value()->view()); + } + dmesgln("USB CDC: Statistics Bitmap: {:#08X}", bm_stats); + dmesgln("USB CDC: Max Segment Size: {}", w_max_segment_size); + if (w_number_mc_filters & (1 << 15)) { + // Perfect filtering is supported + dmesgln("USB CDC: Number of Multicast Filters: Perfect filtering supported"); + } else { + dmesgln("USB CDC: Number of Multicast Filters: {}", w_number_mc_filters & ~(1 << 15)); + } + dmesgln("USB CDC: Number of Power Filters: {}", b_number_power_filters); + } break; + default: + break; + } + return {}; +} + +static ErrorOr dump_cdc_interface(USB::USBConfiguration const& configuration, USB::USBInterface const& interface) +{ + auto subclass_code = interface.descriptor().interface_sub_class_code; + auto protocol_code = interface.descriptor().interface_protocol; + + dmesgln("USB CDC: {}, protocol: ({:#02X}) {}", subclass_code_to_string(CDC::SubclassCode { subclass_code }), protocol_code, protocol_code_to_string(CDC::CommunicationProtocolCode { protocol_code })); + dmesgln("USB CDC: Endpoints: {}", interface.endpoints().size()); + + dmesgln("USB CDC: Functional Descriptors:"); + return configuration.for_each_descriptor_in_interface(interface, [&](ReadonlyBytes raw_descriptor) -> ErrorOr { + auto const& descriptor_header = *reinterpret_cast(raw_descriptor.data()); + if (descriptor_header.descriptor_type == static_cast(CDC::ClassSpecificDescriptorCodes::CS_Interface)) + TRY(dump_cdc_functional_descriptor(configuration, interface, raw_descriptor)); + return IterationDecision::Continue; + }); +} + +static ErrorOr dump_interface(USB::USBConfiguration const& configuration, USB::USBInterface const& interface) +{ + if (interface.descriptor().interface_class_code != USB_CLASS_CDC_DATA && interface.descriptor().interface_class_code != USB_CLASS_COMMUNICATIONS_AND_CDC_CONTROL) + return {}; + + auto& device = const_cast(configuration.device()); // FIXME: Evil!! + auto maybe_interface_string = device.get_string_descriptor(interface.descriptor().interface_string_descriptor_index); + dmesgln("USB CDC: Interface {}.{} ({})", interface.descriptor().interface_id, interface.descriptor().alternate_setting, maybe_interface_string.is_error() ? "No String Descriptor"sv : maybe_interface_string.value()->view()); + + switch (interface.descriptor().interface_class_code) { + case USB_CLASS_CDC_DATA: + return dump_cdc_data_interface(configuration, interface); + case USB_CLASS_COMMUNICATIONS_AND_CDC_CONTROL: + return dump_cdc_interface(configuration, interface); + default: + VERIFY_NOT_REACHED(); + } +} + +struct Function { + USB::USBInterface const& control_interface; + Vector data_interface_ids; + + SubclassCode interface_sub_class() const + { + return SubclassCode { control_interface.descriptor().interface_sub_class_code }; + } + + CommunicationProtocolCode interface_protocol() const + { + return CommunicationProtocolCode { control_interface.descriptor().interface_protocol }; + } +}; + +struct Driver { + StringView name; + SubclassCode interface_sub_class; + CommunicationProtocolCode interface_protocol; + ClassSpecificInterfaceDescriptorCodes required_functional_descriptor; + ErrorOr (*driver_init)(USB::Device&, USB::USBInterface const&, Vector const&); +}; + +constexpr auto drivers = to_array({ + { .name = "ECM"sv, .interface_sub_class = CDC::SubclassCode::EthernetNetworkingControlModel, .interface_protocol = CDC::CommunicationProtocolCode::NoSpecificProtocol, .required_functional_descriptor = CDC::ClassSpecificInterfaceDescriptorCodes::EthernetNetworking, .driver_init = &create_ecm_network_adapter }, +}); + +static ErrorOr select_drivers(USB::Device& device, Vector functions) +{ + bool handled = false; + for (auto const& function : functions) { + for (auto const& driver : drivers) { + if (function.interface_sub_class() != driver.interface_sub_class) + continue; + if (function.interface_protocol() != driver.interface_protocol) + continue; + + // Check for required functional descriptor + bool has_required_descriptor = false; + TRY(function.control_interface.configuration().for_each_descriptor_in_interface(function.control_interface, [&](ReadonlyBytes raw_descriptor) -> ErrorOr { + auto const& descriptor_header = *reinterpret_cast(raw_descriptor.data()); + if (descriptor_header.descriptor_type == static_cast(CDC::ClassSpecificDescriptorCodes::CS_Interface)) { + auto subtype = raw_descriptor[2]; + if (subtype == static_cast(driver.required_functional_descriptor)) { + has_required_descriptor = true; + return IterationDecision::Break; + } + } + return IterationDecision::Continue; + })); + if (!has_required_descriptor) + continue; + + dmesgln("USB CDC: Trying to initialize driver {}", driver.name); + auto result = driver.driver_init(device, function.control_interface, function.data_interface_ids); + if (result.is_error()) { + dmesgln("USB CDC: Failed to initialize driver {}: {}", driver.name, result.error()); + continue; + } + handled = true; + } + } + return handled; +} + +static ErrorOr dump_configuration(USB::Device& device, USB::USBConfiguration const& configuration) +{ + dmesgln("USB CDC: Configuration {}", configuration.descriptor().configuration_value); + Vector functions; + for (auto const& interface : configuration.interfaces()) { + TRY(dump_interface(configuration, interface)); + + if (interface.descriptor().interface_class_code != USB_CLASS_COMMUNICATIONS_AND_CDC_CONTROL) { + continue; + } + + Function function { interface, {} }; + // Find associated data interfaces + // Those are linked via the Union Functional Descriptor + TRY(configuration.for_each_descriptor_in_interface(interface, [&](ReadonlyBytes raw_descriptor) -> ErrorOr { + auto const& descriptor_header = *reinterpret_cast(raw_descriptor.data()); + if (descriptor_header.descriptor_type == static_cast(CDC::ClassSpecificDescriptorCodes::CS_Interface)) { + auto subtype = raw_descriptor[2]; + if (subtype == static_cast(CDC::ClassSpecificInterfaceDescriptorCodes::Union)) { + auto b_master_interface = raw_descriptor[3]; + if (b_master_interface != interface.descriptor().interface_id) { + // Not sure why we would see this, but the spec seems to allow it + return IterationDecision::Continue; + } + Span data_interfaces = raw_descriptor.slice(sizeof(USBDescriptorCommon) + 2); + TRY(function.data_interface_ids.try_extend(data_interfaces)); + } + return IterationDecision::Continue; + } + return IterationDecision::Continue; + })); + TRY(functions.try_append(move(function))); + } + + return select_drivers(device, move(functions)); +} + +ErrorOr CDCDriver::probe(USB::Device& device) +{ + if (device.device_descriptor().device_class != USB_CLASS_COMMUNICATIONS_AND_CDC_CONTROL) { + return ENOTSUP; + } + // Note: SubClass and Protocol should be 0, + // further classification is done on the interface level. + bool handled = false; + dmesgln("USB CDC: Found Device {}:{}", device.device_descriptor().vendor_id, device.device_descriptor().product_id); + for (auto const& configuration : device.configurations()) { + handled |= TRY(dump_configuration(device, configuration)); + } + + if (handled) + return {}; + return ENOTSUP; +} + +} diff --git a/Kernel/Bus/USB/Drivers/HID.cpp b/Kernel/Bus/USB/Drivers/HID.cpp index 8851cd29553e46..fab142432f3a01 100644 --- a/Kernel/Bus/USB/Drivers/HID.cpp +++ b/Kernel/Bus/USB/Drivers/HID.cpp @@ -91,7 +91,7 @@ USB_DEVICE_DRIVER(HIDDriver); void HIDDriver::init() { auto driver = MUST(adopt_nonnull_lock_ref_or_enomem(new (nothrow) HIDDriver())); - USBManagement::register_driver(driver); + MUST(USBManagement::register_driver(driver)); } static constexpr u8 DESCRIPTOR_TYPE_HID = 0x21; @@ -182,7 +182,7 @@ static ErrorOr> initialize_hid_interface(USB::Device ::HID::ReportDescriptorParser report_descriptor_parser { report_descriptor_buffer.span() }; auto parsed_descriptor = TRY(report_descriptor_parser.parse()); - return TRY(HIDInterface::create(device, parsed_descriptor, in_pipe.release_nonnull())); + return TRY(HIDInterface::create(device, move(parsed_descriptor), in_pipe.release_nonnull())); } ErrorOr HIDDriver::probe(USB::Device& device) diff --git a/Kernel/Bus/USB/Drivers/MassStorage/MassStorageDriver.cpp b/Kernel/Bus/USB/Drivers/MassStorage/MassStorageDriver.cpp index ff5efb2a12c64f..8de85629742381 100644 --- a/Kernel/Bus/USB/Drivers/MassStorage/MassStorageDriver.cpp +++ b/Kernel/Bus/USB/Drivers/MassStorage/MassStorageDriver.cpp @@ -23,7 +23,7 @@ USB_DEVICE_DRIVER(MassStorageDriver); void MassStorageDriver::init() { auto driver = MUST(adopt_nonnull_lock_ref_or_enomem(new MassStorageDriver())); - USBManagement::register_driver(driver); + MUST(USBManagement::register_driver(driver)); } ErrorOr MassStorageDriver::probe(USB::Device& device) diff --git a/Kernel/Bus/USB/UHCI/UHCIController.cpp b/Kernel/Bus/USB/UHCI/UHCIController.cpp index 241da49b1eedd9..d42bfe51a8f2f8 100644 --- a/Kernel/Bus/USB/UHCI/UHCIController.cpp +++ b/Kernel/Bus/USB/UHCI/UHCIController.cpp @@ -152,7 +152,7 @@ UNMAP_AFTER_INIT ErrorOr UHCIController::create_structures() m_isochronous_transfer_pool = TRY(MM.allocate_dma_buffer_page("UHCI Isochronous Descriptor Pool"sv, Memory::Region::Access::ReadWrite, Memory::MemoryType::IO)); // Set up the Isochronous Transfer Descriptor list - m_iso_td_list.resize(UHCI_NUMBER_OF_ISOCHRONOUS_TDS); + TRY(m_iso_td_list.try_resize(UHCI_NUMBER_OF_ISOCHRONOUS_TDS)); for (size_t i = 0; i < m_iso_td_list.size(); i++) { auto placement_addr = reinterpret_cast(m_isochronous_transfer_pool->vaddr().get() + (i * sizeof(Kernel::USB::TransferDescriptor))); auto paddr = static_cast(m_isochronous_transfer_pool->physical_page(0)->paddr().get() + (i * sizeof(Kernel::USB::TransferDescriptor))); @@ -343,7 +343,7 @@ ErrorOr UHCIController::initialize_device(USB::Device& device) // Fetch the configuration descriptors from the device auto& configurations = device.configurations({}); - configurations.ensure_capacity(dev_descriptor.num_configurations); + TRY(configurations.try_ensure_capacity(dev_descriptor.num_configurations)); for (u8 configuration = 0u; configuration < dev_descriptor.num_configurations; configuration++) { USBConfigurationDescriptor configuration_descriptor; transfer_length = TRY(device.control_transfer(USB_REQUEST_TRANSFER_DIRECTION_DEVICE_TO_HOST, USB_REQUEST_GET_DESCRIPTOR, (DESCRIPTOR_TYPE_CONFIGURATION << 8u) | configuration, 0, sizeof(USBConfigurationDescriptor), &configuration_descriptor)); diff --git a/Kernel/Bus/USB/USBClasses.h b/Kernel/Bus/USB/USBClasses.h index a7ee20a3bf22d7..fa13dbbd75bf07 100644 --- a/Kernel/Bus/USB/USBClasses.h +++ b/Kernel/Bus/USB/USBClasses.h @@ -20,7 +20,7 @@ static constexpr u8 USB_CLASS_IMAGE = 0x06; static constexpr u8 USB_CLASS_PRINTER = 0x07; static constexpr u8 USB_CLASS_MASS_STORAGE = 0x08; static constexpr u8 USB_CLASS_HUB = 0x09; -static constexpr u8 USB_CLASS_CDC_DATE = 0x0A; +static constexpr u8 USB_CLASS_CDC_DATA = 0x0A; static constexpr u8 USB_CLASS_SMART_CARD = 0x0B; static constexpr u8 USB_CLASS_CONTENT_SECURITY = 0x0D; static constexpr u8 USB_CLASS_VIDEO = 0x0E; diff --git a/Kernel/Bus/USB/USBConfiguration.cpp b/Kernel/Bus/USB/USBConfiguration.cpp index 034bc0ecde46b4..dfdc55b0ee75b4 100644 --- a/Kernel/Bus/USB/USBConfiguration.cpp +++ b/Kernel/Bus/USB/USBConfiguration.cpp @@ -18,7 +18,7 @@ USBConfiguration::USBConfiguration(USBConfiguration const& other) : m_device(other.m_device) , m_descriptor(other.m_descriptor) , m_descriptor_index(other.m_descriptor_index) - , m_interfaces(other.m_interfaces) + , m_interfaces(other.m_interfaces.clone().release_value_but_fixme_should_propagate_errors()) { // FIXME: This can definitely OOM for (auto& interface : m_interfaces) diff --git a/Kernel/Bus/USB/USBConfiguration.h b/Kernel/Bus/USB/USBConfiguration.h index 214ab91179d461..954a1cea6711fa 100644 --- a/Kernel/Bus/USB/USBConfiguration.h +++ b/Kernel/Bus/USB/USBConfiguration.h @@ -25,7 +25,7 @@ class USBConfiguration { , m_descriptor(descriptor) , m_descriptor_index(descriptor_index) { - m_interfaces.ensure_capacity(descriptor.number_of_interfaces); + m_interfaces.try_ensure_capacity(descriptor.number_of_interfaces).release_value_but_fixme_should_propagate_errors(); } private: diff --git a/Kernel/Bus/USB/USBDevice.cpp b/Kernel/Bus/USB/USBDevice.cpp index de537f81fa6f75..189142df78132f 100644 --- a/Kernel/Bus/USB/USBDevice.cpp +++ b/Kernel/Bus/USB/USBDevice.cpp @@ -4,6 +4,7 @@ * SPDX-License-Identifier: BSD-2-Clause */ +#include #include #include #include @@ -13,6 +14,7 @@ #include #include #include +#include #include namespace Kernel::USB { @@ -74,8 +76,7 @@ Device::Device(Device const& device) , m_controller(device.controller()) , m_hub(device.hub()) { - // FIXME: This can definitely OOM - m_configurations.ensure_capacity(device.configurations().size()); + m_configurations.try_ensure_capacity(device.configurations().size()).release_value_but_fixme_should_propagate_errors(); for (auto const& configuration : device.configurations()) { m_configurations.unchecked_append(configuration.copy()); m_configurations.last().set_device({}, *this); @@ -92,6 +93,69 @@ void Device::set_default_pipe(NonnullOwnPtr pipe) m_default_pipe = move(pipe); } +ErrorOr> Device::get_string_descriptor(u8 descriptor_index) +{ + // Index 0 usually means no string descriptor + // and would be 0 the list of available languages, which usually isn't a valid string + if (descriptor_index == 0) + return KString::try_create(""sv); + + Array buffer; + // Get Available languages + // FIXME: We should likely cache this + auto transfer_length = TRY(control_transfer(USB_REQUEST_TRANSFER_DIRECTION_DEVICE_TO_HOST | USB_REQUEST_TYPE_STANDARD | USB_REQUEST_RECIPIENT_DEVICE, + USB_REQUEST_GET_DESCRIPTOR, (DESCRIPTOR_TYPE_STRING << 8) | 0, 0, buffer.size(), buffer.data())); + + if (transfer_length < sizeof(USBDescriptorCommon)) { + dmesgln("USB Device: Could not query supported languages"); + return EIO; + } + // After the header there is list of u16 language IDs, + // Check for the preferred language ID, or take the first one + auto const* header = reinterpret_cast(buffer.data()); + // FIXME: This should likely be customizable/respect the locale + constexpr u16 preferred_language_id = 0x0409; // English (US) + + if (header->length == sizeof(USBDescriptorCommon)) { + dmesgln("USB Device: No supported languages found"); + return ENOTSUP; + } + + constexpr size_t header_size_in_u16 = sizeof(USBDescriptorCommon) / sizeof(u16); + Span languages = buffer.span().slice(header_size_in_u16, min((header->length / sizeof(u16)) - header_size_in_u16, transfer_length / sizeof(u16))); + u16 lang_id; + if (languages.contains_slow(preferred_language_id)) + lang_id = preferred_language_id; + else + lang_id = languages[0]; + + transfer_length = TRY(control_transfer(USB_REQUEST_TRANSFER_DIRECTION_DEVICE_TO_HOST | USB_REQUEST_TYPE_STANDARD | USB_REQUEST_RECIPIENT_DEVICE, + USB_REQUEST_GET_DESCRIPTOR, (DESCRIPTOR_TYPE_STRING << 8) | descriptor_index, lang_id, sizeof(buffer), buffer.data())); + + if (transfer_length < 2) + return KString::try_create(""sv); + + header = reinterpret_cast(buffer.data()); + if (header->descriptor_type != DESCRIPTOR_TYPE_STRING) { + dmesgln("USB Device: Invalid string descriptor received, expected type {} but got {}", DESCRIPTOR_TYPE_STRING, header->descriptor_type); + return KString::try_create(""sv); + } + auto const descriptor_length = min(header->length, transfer_length); + + // Trailing data is a UTF-16LE encoded string + // FIXME: Let's just assume ascii with zero high bytes for now + Span string_data = buffer.span().slice(header_size_in_u16, (descriptor_length / sizeof(u16)) - header_size_in_u16); + char* ascii_view; + auto string = TRY(KString::try_create_uninitialized(string_data.size(), ascii_view)); + for (auto ch : string_data) { + if (ch > 0xFF) + *ascii_view++ = '?'; + else + *ascii_view++ = static_cast(ch); + } + return string; +} + ErrorOr Device::control_transfer(u8 request_type, u8 request, u16 value, u16 index, u16 length, void* data) { return TRY(m_default_pipe->submit_control_transfer(request_type, request, value, index, length, data)); diff --git a/Kernel/Bus/USB/USBDevice.h b/Kernel/Bus/USB/USBDevice.h index 69734c9b717e13..1771521522cfa3 100644 --- a/Kernel/Bus/USB/USBDevice.h +++ b/Kernel/Bus/USB/USBDevice.h @@ -14,6 +14,7 @@ #include #include #include +#include #include namespace Kernel { @@ -60,6 +61,8 @@ class Device : public AtomicRefCounted { USBDeviceDescriptor const& device_descriptor() const { return m_device_descriptor; } + ErrorOr> get_string_descriptor(u8 descriptor_index); + USBController& controller() { return *m_controller; } USBController const& controller() const { return *m_controller; } @@ -77,6 +80,7 @@ class Device : public AtomicRefCounted { m_driver = nullptr; } + ErrorOr set_configuration(USBConfiguration const& configuration); ErrorOr set_configuration_and_interface(USBInterface const& interface); RecursiveSpinlockProtected, LockRank::None>& sysfs_device_info_node(Badge) { return m_sysfs_device_info_node; } @@ -108,7 +112,6 @@ class Device : public AtomicRefCounted { protected: void set_default_pipe(NonnullOwnPtr pipe); - ErrorOr set_configuration(USBConfiguration const& configuration); u8 m_device_port { 0 }; // What port is this device attached to. NOTE: This is 1-based. DeviceSpeed m_device_speed; // What speed is this device running at diff --git a/Kernel/Bus/USB/USBInterface.h b/Kernel/Bus/USB/USBInterface.h index 3367bc4c62052c..def964435723c9 100644 --- a/Kernel/Bus/USB/USBInterface.h +++ b/Kernel/Bus/USB/USBInterface.h @@ -24,6 +24,14 @@ class USBInterface final { { } + USBInterface(USBInterface const& other) + : m_configuration(other.m_configuration) + , m_descriptor(other.m_descriptor) + , m_endpoint_descriptors(other.m_endpoint_descriptors.clone().release_value_but_fixme_should_propagate_errors()) + , m_descriptor_offset(other.m_descriptor_offset) + { + } + ErrorOr add_endpoint_descriptor(Badge, USBEndpointDescriptor endpoint_descriptor) { return m_endpoint_descriptors.try_empend(endpoint_descriptor); } Vector const& endpoints() const { return m_endpoint_descriptors; } diff --git a/Kernel/Bus/USB/USBManagement.cpp b/Kernel/Bus/USB/USBManagement.cpp index 14b3ce61da8b82..49cee155da7641 100644 --- a/Kernel/Bus/USB/USBManagement.cpp +++ b/Kernel/Bus/USB/USBManagement.cpp @@ -20,10 +20,10 @@ UNMAP_AFTER_INIT USBManagement::USBManagement() SysFSUSBBusDirectory::initialize(); } -void USBManagement::register_driver(NonnullLockRefPtr driver) +ErrorOr USBManagement::register_driver(NonnullLockRefPtr driver) { dbgln_if(USB_DEBUG, "Registering driver {}", driver->name()); - s_available_drivers->append(driver); + return s_available_drivers->try_append(driver); } LockRefPtr USBManagement::get_driver_by_name(StringView name) diff --git a/Kernel/Bus/USB/USBManagement.h b/Kernel/Bus/USB/USBManagement.h index dd31ddabdcfef2..d77fe39cc64eab 100644 --- a/Kernel/Bus/USB/USBManagement.h +++ b/Kernel/Bus/USB/USBManagement.h @@ -19,7 +19,7 @@ class USBManagement { USBManagement(); static USBManagement& the(); - static void register_driver(NonnullLockRefPtr driver); + static ErrorOr register_driver(NonnullLockRefPtr driver); static LockRefPtr get_driver_by_name(StringView name); static void unregister_driver(NonnullLockRefPtr driver); diff --git a/Kernel/Bus/USB/xHCI/DeviceTreexHCIController.cpp b/Kernel/Bus/USB/xHCI/DeviceTreexHCIController.cpp index 6b5696df300a48..b3402ffa198f5b 100644 --- a/Kernel/Bus/USB/xHCI/DeviceTreexHCIController.cpp +++ b/Kernel/Bus/USB/xHCI/DeviceTreexHCIController.cpp @@ -44,21 +44,7 @@ DEVICETREE_DRIVER(DeviceTreexHCIControllerDriver, compatibles_array); ErrorOr DeviceTreexHCIControllerDriver::probe(DeviceTree::Device const& device, StringView) const { auto registers_resource = TRY(device.get_resource(0)); - auto interrupt = TRY(device.node().interrupts(DeviceTree::get()))[0]; - - // FIXME: Don't depend on a specific interrupt descriptor format and implement proper devicetree interrupt mapping/translation. - if (!interrupt.domain_root->is_compatible_with("arm,gic-400"sv) && !interrupt.domain_root->is_compatible_with("arm,cortex-a15-gic"sv)) - return ENOTSUP; - if (interrupt.interrupt_identifier.size() != 3 * sizeof(BigEndian)) - return ENOTSUP; - - // The interrupt type is in the first cell. It should be 0 for SPIs. - if (reinterpret_cast const*>(interrupt.interrupt_identifier.data())[0] != 0) - return ENOTSUP; - - // The interrupt number is in the second cell. - // GIC interrupts 32-1019 are for SPIs, so add 32 to get the GIC interrupt ID. - auto interrupt_number = (reinterpret_cast const*>(interrupt.interrupt_identifier.data())[1]) + 32; + auto interrupt_number = TRY(device.get_interrupt_number(0)); auto controller = TRY(DeviceTreexHCIController::try_to_initialize(registers_resource, device.node_name(), interrupt_number)); USB::USBManagement::the().add_controller(controller); diff --git a/Kernel/Bus/USB/xHCI/xHCIController.cpp b/Kernel/Bus/USB/xHCI/xHCIController.cpp index 92957280a4a6c7..9b6b1ff32719f3 100644 --- a/Kernel/Bus/USB/xHCI/xHCIController.cpp +++ b/Kernel/Bus/USB/xHCI/xHCIController.cpp @@ -367,9 +367,20 @@ void xHCIController::enqueue_command(TransferRequestBlock& transfer_request_bloc void xHCIController::execute_command(TransferRequestBlock& transfer_request_block) { - SpinlockLocker const locker(m_command_lock); + MutexLocker const locker(m_command_mutex); + enqueue_command(transfer_request_block); - m_command_completion_queue.wait_forever(); + + // If the current process isn't a kernel process, we could receive signals and would therefore need to handle wait_until failing. + VERIFY(Process::current().is_kernel_process()); + + MUST(m_command_completion_queue.wait_until(m_current_command_complete, [](bool& current_command_complete) { + if (!current_command_complete) + return false; + current_command_complete = false; + return true; + })); + transfer_request_block = m_command_result_transfer_request_block; } @@ -514,7 +525,7 @@ ErrorOr xHCIController::initialize_device(USB::Device& device) device.set_controller_identifier({}, slot); auto& slot_state = m_slots_state[slot - 1]; - SpinlockLocker const locker(slot_state.lock); + MutexLocker const input_context_locker(slot_state.input_context_mutex); VERIFY(!slot_state.input_context_region); // Prevent trying to initialize an already initialized device // 5. After successfully obtaining a Device Slot, system software shall initialize the data structures associated with the slot as described in section 4.3.3. @@ -596,16 +607,20 @@ ErrorOr xHCIController::initialize_device(USB::Device& device) slot_context->parent_hub_slot_id = device.hub()->controller_identifier(); } + auto control_endpoint_ring_region = TRY(MM.allocate_dma_buffer_pages(MUST(Memory::page_round_up(endpoint_ring_size * sizeof(TransferRequestBlock))), "xHCI Endpoint Rings"sv, Memory::Region::ReadWrite, Memory::MemoryType::IO)); + auto control_endpoint_ring_address = control_endpoint_ring_region->physical_page(0)->paddr().get(); + // 4. Allocate and initialize the Transfer Ring for the Default Control Endpoint. // Refer to section 4.9 for TRB Ring initialization requirements and to section 6.4 for the formats of TRBs // FIXME: Synchronize DMA buffer accesses correctly and set the MemoryType to NonCacheable. - slot_state.endpoint_rings[0].region = TRY(MM.allocate_dma_buffer_pages(MUST(Memory::page_round_up(endpoint_ring_size * sizeof(TransferRequestBlock))), "xHCI Endpoint Rings"sv, Memory::Region::ReadWrite, Memory::MemoryType::IO)); - auto* endpoint_ring_memory = slot_state.endpoint_rings[0].ring_vaddr(); - endpoint_ring_memory[endpoint_ring_size - 1].generic.transfer_request_block_type = TransferRequestBlock::TRBType::Link; - auto endpoint_ring_address = slot_state.endpoint_rings[0].ring_paddr(); - endpoint_ring_memory[endpoint_ring_size - 1].link.ring_segment_pointer_low = endpoint_ring_address; - endpoint_ring_memory[endpoint_ring_size - 1].link.ring_segment_pointer_high = endpoint_ring_address >> 32; - endpoint_ring_memory[endpoint_ring_size - 1].link.toggle_cycle = 1; + slot_state.endpoint_rings[0].with([&control_endpoint_ring_region, control_endpoint_ring_address](auto& control_endpoint_ring) { + control_endpoint_ring.region = move(control_endpoint_ring_region); + auto* endpoint_ring_memory = control_endpoint_ring.ring_vaddr(); + endpoint_ring_memory[endpoint_ring_size - 1].generic.transfer_request_block_type = TransferRequestBlock::TRBType::Link; + endpoint_ring_memory[endpoint_ring_size - 1].link.ring_segment_pointer_low = control_endpoint_ring_address; + endpoint_ring_memory[endpoint_ring_size - 1].link.ring_segment_pointer_high = control_endpoint_ring_address >> 32; + endpoint_ring_memory[endpoint_ring_size - 1].link.toggle_cycle = 1; + }); // 5. Initialize the Input default control Endpoint 0 Context (6.2.3). auto* endpoint_context = input_endpoint_context(slot, 0, Pipe::Direction::Bidirectional); @@ -631,8 +646,8 @@ ErrorOr xHCIController::initialize_device(USB::Device& device) // * Max Burst Size = 0. endpoint_context->max_burst_size = 0; // * TR Dequeue Pointer = Start address of first segment of the Default Control Endpoint Transfer Ring. - endpoint_context->transfer_ring_dequeue_pointer_low = endpoint_ring_address >> 4; - endpoint_context->transfer_ring_dequeue_pointer_high = endpoint_ring_address >> 32; + endpoint_context->transfer_ring_dequeue_pointer_low = control_endpoint_ring_address >> 4; + endpoint_context->transfer_ring_dequeue_pointer_high = control_endpoint_ring_address >> 32; // * Dequeue Cycle State (DCS) = 1. Reflects Cycle bit state for valid TRBs written by software. endpoint_context->dequeue_cycle_state = 1; // * Interval = 0. @@ -715,7 +730,7 @@ ErrorOr xHCIController::initialize_device(USB::Device& device) // Fetch the configuration descriptors from the device auto& configurations = device.configurations({}); - configurations.ensure_capacity(dev_descriptor.num_configurations); + TRY(configurations.try_ensure_capacity(dev_descriptor.num_configurations)); for (u8 configuration = 0u; configuration < dev_descriptor.num_configurations; configuration++) { USBConfigurationDescriptor configuration_descriptor; transfer_length = TRY(device.control_transfer(USB_REQUEST_TRANSFER_DIRECTION_DEVICE_TO_HOST, USB_REQUEST_GET_DESCRIPTOR, (DESCRIPTOR_TYPE_CONFIGURATION << 8u) | configuration, 0, sizeof(USBConfigurationDescriptor), &configuration_descriptor)); @@ -740,9 +755,14 @@ ErrorOr xHCIController::enqueue_transfer(u8 slot, u8 endpoint, Pipe::Direc { VERIFY(transfer_request_blocks.size() > 0); VERIFY(transfer_request_blocks.size() < endpoint_ring_size); - SpinlockLocker const locker(m_slots_state[slot - 1].lock); - auto& endpoint_ring = m_slots_state[slot - 1].endpoint_rings[endpoint_index(endpoint, direction) - 1]; + return m_slots_state[slot - 1].endpoint_rings[endpoint_index(endpoint, direction) - 1].with([this, slot, endpoint, direction, &transfer_request_blocks, &pending_transfer](auto& endpoint_ring) -> ErrorOr { + return enqueue_transfer_impl(slot, endpoint, direction, transfer_request_blocks, pending_transfer, endpoint_ring); + }); +} + +ErrorOr xHCIController::enqueue_transfer_impl(u8 slot, u8 endpoint, Pipe::Direction direction, Span transfer_request_blocks, PendingTransfer& pending_transfer, EndpointRing& endpoint_ring) +{ VERIFY(endpoint_ring.region); if (transfer_request_blocks.size() > endpoint_ring.free_transfer_request_blocks) return ENOBUFS; @@ -842,7 +862,12 @@ ErrorOr xHCIController::submit_control_transfer(Transfer& transfer) SyncPendingTransfer pending_transfer; TRY(enqueue_transfer(slot, 0, Pipe::Direction::Bidirectional, { transfer_request_blocks, trb_index }, pending_transfer)); - pending_transfer.wait_queue.wait_forever(); + MUST(pending_transfer.wait_queue.wait_until(pending_transfer.transfer_done, [](bool& transfer_done) { + if (!transfer_done) + return false; + transfer_done = false; + return true; + })); VERIFY(!pending_transfer.endpoint_list_node.is_in_list()); if (pending_transfer.completion_code == TransferRequestBlock::CompletionCode::Stall_Error) { @@ -879,11 +904,28 @@ ErrorOr> xHCIController::prepare_normal_transfer(Tr auto const& device = transfer.pipe().device(); auto const slot = device.controller_identifier(); + // Short circuit for empty transfers + // Otherwise we'd accidentally not create any TRBs at all... + if (transfer.transfer_data_size() == 0) { + TransferRequestBlock empty_trb {}; + empty_trb.normal.data_buffer_pointer_low = 0; + empty_trb.normal.data_buffer_pointer_high = 0; + empty_trb.normal.transfer_request_block_transfer_length = 0; + empty_trb.normal.transfer_descriptor_size = 0; + empty_trb.normal.interrupter_target = 0; + empty_trb.normal.interrupt_on_completion = 1; + empty_trb.normal.transfer_request_block_type = TransferRequestBlock::TRBType::Normal; + Vector empty_trb_vector; + TRY(empty_trb_vector.try_append(empty_trb)); + return empty_trb_vector; + } + u32 max_burst_payload = 0; { auto endpoint_id = endpoint_index(transfer.pipe().endpoint_number(), transfer.pipe().direction()); - SpinlockLocker locker(m_slots_state[slot - 1].lock); - max_burst_payload = m_slots_state[slot - 1].endpoint_rings[endpoint_id - 1].max_burst_payload; + m_slots_state[slot - 1].endpoint_rings[endpoint_id - 1].with([&max_burst_payload](auto const& endpoint_ring) { + max_burst_payload = endpoint_ring.max_burst_payload; + }); } VERIFY(max_burst_payload > 0); @@ -928,7 +970,12 @@ ErrorOr xHCIController::submit_bulk_transfer(Transfer& transfer) SyncPendingTransfer pending_transfer; TRY(enqueue_transfer(transfer.pipe().device().controller_identifier(), transfer.pipe().endpoint_number(), transfer.pipe().direction(), transfer_request_blocks, pending_transfer)); - pending_transfer.wait_queue.wait_forever(); + MUST(pending_transfer.wait_queue.wait_until(pending_transfer.transfer_done, [](bool& transfer_done) { + if (!transfer_done) + return false; + transfer_done = false; + return true; + })); VERIFY(!pending_transfer.endpoint_list_node.is_in_list()); if (pending_transfer.completion_code == TransferRequestBlock::CompletionCode::Stall_Error) @@ -977,19 +1024,22 @@ ErrorOr xHCIController::reset_pipe(USB::Device& device, USB::Pipe& pipe) } auto& slot_state = m_slots_state[slot - 1]; - auto& endpoint_ring = slot_state.endpoint_rings[endpoint_id - 1]; - // Issue a Set TR Dequeue Pointer Command, clear the endpoint state and reference the TRB to start. + TRY(slot_state.endpoint_rings[endpoint_id - 1].with([slot, endpoint_id, this](auto& endpoint_ring) -> ErrorOr { + // Issue a Set TR Dequeue Pointer Command, clear the endpoint state and reference the TRB to start. + + // TODO: Set the Stream ID and Stream Context Type if streams are enabled for the endpoint once we support streams. + TRY(set_tr_dequeue_pointer(slot, endpoint_id, 0, 0, endpoint_ring.ring_paddr(), 1)); - // TODO: Set the Stream ID and Stream Context Type if streams are enabled for the endpoint once we support streams. - TRY(set_tr_dequeue_pointer(slot, endpoint_id, 0, 0, endpoint_ring.ring_paddr(), 1)); + endpoint_ring.enqueue_index = 0; + endpoint_ring.pending_transfers.clear(); + endpoint_ring.producer_cycle_state = 1; + endpoint_ring.free_transfer_request_blocks = endpoint_ring_size - 1; // -1 to exclude the Link TRB + for (size_t i = 0; i < endpoint_ring_size - 1; i++) + endpoint_ring.ring_vaddr()[i] = {}; - endpoint_ring.enqueue_index = 0; - endpoint_ring.pending_transfers.clear(); - endpoint_ring.producer_cycle_state = 1; - endpoint_ring.free_transfer_request_blocks = endpoint_ring_size - 1; // -1 to exclude the Link TRB - for (size_t i = 0; i < endpoint_ring_size - 1; i++) - endpoint_ring.ring_vaddr()[i] = {}; + return {}; + })); // Ring Doorbell to restart the pipe. ring_endpoint_doorbell(slot, pipe.endpoint_number(), pipe.direction()); @@ -1002,10 +1052,16 @@ ErrorOr xHCIController::initialize_endpoint_if_needed(Pipe const& pipe) VERIFY(pipe.endpoint_number() != 0); // Endpoint 0 is manually initialized during device initialization auto const slot = pipe.device().controller_identifier(); auto& slot_state = m_slots_state[slot - 1]; - SpinlockLocker locker(slot_state.lock); + + // Locking this mutex also ensures that this function can't be run on the same device in parallel. + MutexLocker const input_context_locker(slot_state.input_context_mutex); + VERIFY(slot_state.input_context_region); auto endpoint_id = endpoint_index(pipe.endpoint_number(), pipe.direction()); - auto& endpoint_ring = slot_state.endpoint_rings[endpoint_id - 1]; + + // We don't need any proper locking in this function yet, since transfers on this endpoint aren't possible yet, so no one else should be using this endpoint ring. + auto& endpoint_ring = slot_state.endpoint_rings[endpoint_id - 1].with([](auto& endpoint_ring) -> auto& { return endpoint_ring; }); + if (endpoint_ring.region) return {}; // Already initialized @@ -1299,7 +1355,8 @@ void xHCIController::handle_interrupt(u16 interrupter_id) return; } if (usb_status.event_interrupt) { - m_event_queue.wake_all(); + m_events_pending.with([](bool& events_pending) { events_pending = true; }); + m_event_queue.notify_one(); return; } } @@ -1309,59 +1366,65 @@ void xHCIController::handle_transfer_event(TransferRequestBlock const& transfer_ auto slot = transfer_request_block.transfer_event.slot_id; VERIFY(slot > 0 && slot <= m_device_slots); auto& slot_state = m_slots_state[slot - 1]; - SpinlockLocker const locker(slot_state.lock); auto endpoint = transfer_request_block.transfer_event.endpoint_id; VERIFY(endpoint > 0 && endpoint <= max_endpoints); - auto& endpoint_ring = slot_state.endpoint_rings[endpoint - 1]; - VERIFY(endpoint_ring.region); - - if (transfer_request_block.transfer_event.completion_code != TransferRequestBlock::CompletionCode::Success - && transfer_request_block.transfer_event.completion_code != TransferRequestBlock::CompletionCode::Short_Packet) - dmesgln_xhci("Transfer error on slot {} endpoint {}: {}", slot, endpoint, enum_to_string(transfer_request_block.transfer_event.completion_code)); - - VERIFY(transfer_request_block.transfer_event.event_data == 0); // The Pointer points to the interrupting TRB - auto transfer_request_block_pointer = ((u64)transfer_request_block.transfer_event.transfer_request_block_pointer_high << 32) | transfer_request_block.transfer_event.transfer_request_block_pointer_low; - VERIFY(transfer_request_block_pointer % sizeof(TransferRequestBlock) == 0); - if (transfer_request_block_pointer < endpoint_ring.ring_paddr() || (transfer_request_block_pointer - endpoint_ring.ring_paddr()) > (endpoint_ring_size * sizeof(TransferRequestBlock))) { - dmesgln_xhci("Transfer event on slot {} endpoint {} points to unknown TRB", slot, endpoint); - return; - } - auto transfer_request_block_index = (transfer_request_block_pointer - endpoint_ring.ring_paddr()) / sizeof(TransferRequestBlock); - for (auto& pending_transfer : endpoint_ring.pending_transfers) { - auto freed_transfer_request_blocks = 0; - if (pending_transfer.start_index <= pending_transfer.end_index) { - if (pending_transfer.start_index > transfer_request_block_index || transfer_request_block_index > pending_transfer.end_index) - continue; - freed_transfer_request_blocks = pending_transfer.end_index - pending_transfer.start_index + 1; - } else { - if (pending_transfer.start_index > transfer_request_block_index && transfer_request_block_index > pending_transfer.end_index) - continue; - freed_transfer_request_blocks = (endpoint_ring_size - pending_transfer.start_index) + pending_transfer.end_index; + slot_state.endpoint_rings[endpoint - 1].with([&transfer_request_block, slot, endpoint, this](auto& endpoint_ring) { + VERIFY(endpoint_ring.region); + + if (transfer_request_block.transfer_event.completion_code != TransferRequestBlock::CompletionCode::Success + && transfer_request_block.transfer_event.completion_code != TransferRequestBlock::CompletionCode::Short_Packet) + dmesgln_xhci("Transfer error on slot {} endpoint {}: {}", slot, endpoint, enum_to_string(transfer_request_block.transfer_event.completion_code)); + + VERIFY(transfer_request_block.transfer_event.event_data == 0); // The Pointer points to the interrupting TRB + auto transfer_request_block_pointer = ((u64)transfer_request_block.transfer_event.transfer_request_block_pointer_high << 32) | transfer_request_block.transfer_event.transfer_request_block_pointer_low; + VERIFY(transfer_request_block_pointer % sizeof(TransferRequestBlock) == 0); + if (transfer_request_block_pointer < endpoint_ring.ring_paddr() || (transfer_request_block_pointer - endpoint_ring.ring_paddr()) > (endpoint_ring_size * sizeof(TransferRequestBlock))) { + dmesgln_xhci("Transfer event on slot {} endpoint {} points to unknown TRB", slot, endpoint); + return; } - endpoint_ring.free_transfer_request_blocks += freed_transfer_request_blocks; - pending_transfer.endpoint_list_node.remove(); - if (endpoint_ring.type == Pipe::Type::Control || endpoint_ring.type == Pipe::Type::Bulk) { - auto& sync_pending_transfer = static_cast(pending_transfer); - sync_pending_transfer.completion_code = transfer_request_block.transfer_event.completion_code; - sync_pending_transfer.remainder = transfer_request_block.transfer_event.transfer_request_block_transfer_length; - full_memory_fence(); - sync_pending_transfer.wait_queue.wake_all(); - } else { - auto& periodic_pending_transfer = static_cast(pending_transfer); - periodic_pending_transfer.original_transfer->invoke_async_callback(); - // Reschedule the periodic transfer (NOTE: We MUST() here since a re-enqueue should never fail) - MUST(enqueue_transfer(slot, periodic_pending_transfer.original_transfer->pipe().endpoint_number(), periodic_pending_transfer.original_transfer->pipe().direction(), periodic_pending_transfer.transfer_request_blocks, periodic_pending_transfer)); + auto transfer_request_block_index = (transfer_request_block_pointer - endpoint_ring.ring_paddr()) / sizeof(TransferRequestBlock); + for (auto& pending_transfer : endpoint_ring.pending_transfers) { + auto freed_transfer_request_blocks = 0; + if (pending_transfer.start_index <= pending_transfer.end_index) { + if (pending_transfer.start_index > transfer_request_block_index || transfer_request_block_index > pending_transfer.end_index) + continue; + freed_transfer_request_blocks = pending_transfer.end_index - pending_transfer.start_index + 1; + } else { + if (pending_transfer.start_index > transfer_request_block_index && transfer_request_block_index > pending_transfer.end_index) + continue; + freed_transfer_request_blocks = (endpoint_ring_size - pending_transfer.start_index) + pending_transfer.end_index; + } + endpoint_ring.free_transfer_request_blocks += freed_transfer_request_blocks; + pending_transfer.endpoint_list_node.remove(); + if (endpoint_ring.type == Pipe::Type::Control || endpoint_ring.type == Pipe::Type::Bulk) { + auto& sync_pending_transfer = static_cast(pending_transfer); + sync_pending_transfer.completion_code = transfer_request_block.transfer_event.completion_code; + sync_pending_transfer.remainder = transfer_request_block.transfer_event.transfer_request_block_transfer_length; + full_memory_fence(); + sync_pending_transfer.transfer_done.with([](bool& transfer_done) { transfer_done = true; }); + sync_pending_transfer.wait_queue.notify_one(); + } else { + auto& periodic_pending_transfer = static_cast(pending_transfer); + periodic_pending_transfer.original_transfer->invoke_async_callback(); + // Reschedule the periodic transfer (NOTE: We MUST() here since a re-enqueue should never fail) + MUST(enqueue_transfer_impl(slot, periodic_pending_transfer.original_transfer->pipe().endpoint_number(), periodic_pending_transfer.original_transfer->pipe().direction(), periodic_pending_transfer.transfer_request_blocks, periodic_pending_transfer, endpoint_ring)); + } + return; } - return; - } - dmesgln_xhci("Transfer event on slot {} endpoint {} points to unowned TRB", slot, endpoint); + dmesgln_xhci("Transfer event on slot {} endpoint {} points to unowned TRB", slot, endpoint); + }); } void xHCIController::event_handling_thread() { while (!Process::current().is_dying()) { - m_event_queue.wait_forever("xHCI"sv); + MUST(m_event_queue.wait_until(m_events_pending, [](bool& events_pending) { + if (!events_pending) + return false; + events_pending = false; + return true; + })); bool header_printed = false; @@ -1396,7 +1459,8 @@ void xHCIController::event_handling_thread() // active command result. m_command_result_transfer_request_block = m_event_ring_segment[m_event_ring_dequeue_index]; full_memory_fence(); - m_command_completion_queue.wake_all(); + m_current_command_complete.with([](bool& current_command_complete) { current_command_complete = true; }); + m_command_completion_queue.notify_one(); break; case TransferRequestBlock::TRBType::Port_Status_Change_Event: dbgln_if(XHCI_DEBUG, "Port status change detected by controller"); diff --git a/Kernel/Bus/USB/xHCI/xHCIController.h b/Kernel/Bus/USB/xHCI/xHCIController.h index a9a5fb16c73afd..81fdacdf119915 100644 --- a/Kernel/Bus/USB/xHCI/xHCIController.h +++ b/Kernel/Bus/USB/xHCI/xHCIController.h @@ -14,7 +14,7 @@ #include #include #include -#include +#include namespace Kernel::USB::xHCI { @@ -96,7 +96,8 @@ class xHCIController u32 end_index { 0 }; }; struct SyncPendingTransfer : public PendingTransfer { - DeprecatedWaitQueue wait_queue; + WaitQueue wait_queue; + SpinlockProtected transfer_done { false }; TransferRequestBlock::CompletionCode completion_code { TransferRequestBlock::CompletionCode::Invalid }; u32 remainder { 0 }; }; @@ -119,10 +120,12 @@ class xHCIController static constexpr size_t max_endpoints = 31; static constexpr size_t endpoint_ring_size = PAGE_SIZE / sizeof(TransferRequestBlock); struct SlotState { - RecursiveSpinlock lock; + Mutex input_context_mutex; OwnPtr input_context_region; + OwnPtr device_context_region; - Array endpoint_rings; + + Array, max_endpoints> endpoint_rings; }; static u8 endpoint_index(u8 endpoint, Pipe::Direction direction) @@ -139,7 +142,10 @@ class xHCIController u8* input_context(u8 slot, u8 index) const { - auto* base = m_slots_state[slot - 1].input_context_region->vaddr().as_ptr(); + auto const& slot_state = m_slots_state[slot - 1]; + VERIFY(slot_state.input_context_mutex.is_locked()); + + auto* base = slot_state.input_context_region->vaddr().as_ptr(); return base + (context_entry_size() * index); } @@ -160,20 +166,21 @@ class xHCIController size_t device_context_size() const { return context_entry_size() * 32; } - u8* device_context(u8 slot, u8 index) const + u8 const* device_context(u8 slot, u8 index) const { - auto* base = m_slots_state[slot - 1].device_context_region->vaddr().as_ptr(); + auto const& slot_state = m_slots_state[slot - 1]; + auto* base = slot_state.device_context_region->vaddr().as_ptr(); return base + (context_entry_size() * index); } - SlotContext* device_slot_context(u8 slot) const + SlotContext const* device_slot_context(u8 slot) const { - return reinterpret_cast(device_context(slot, 0)); + return reinterpret_cast(device_context(slot, 0)); } - EndpointContext* device_endpoint_context(u8 slot, u8 endpoint, Pipe::Direction direction) const + EndpointContext const* device_endpoint_context(u8 slot, u8 endpoint, Pipe::Direction direction) const { - return reinterpret_cast(device_context(slot, endpoint_index(endpoint, direction))); + return reinterpret_cast(device_context(slot, endpoint_index(endpoint, direction))); } void ring_doorbell(u8 doorbell, u8 doorbell_target); @@ -200,6 +207,7 @@ class xHCIController ErrorOr set_tr_dequeue_pointer(u8 slot, u8 endpoint, u8 stream_context_type, u16 stream, u64 new_tr_dequeue_pointer, u8 dequeue_cycle_state); ErrorOr enqueue_transfer(u8 slot, u8 endpoint, Pipe::Direction direction, Span, PendingTransfer&); + ErrorOr enqueue_transfer_impl(u8 slot, u8 endpoint, Pipe::Direction direction, Span, PendingTransfer&, EndpointRing&); void handle_transfer_event(TransferRequestBlock const&); ErrorOr> prepare_normal_transfer(Transfer& transfer); @@ -211,7 +219,8 @@ class xHCIController DoorbellRegisters volatile& m_doorbell_registers; RefPtr m_process; - DeprecatedWaitQueue m_event_queue; + SpinlockProtected m_events_pending { false }; + WaitQueue m_event_queue; bool m_using_message_signalled_interrupts { false }; bool m_large_contexts { false }; @@ -222,8 +231,9 @@ class xHCIController Vector> m_active_periodic_transfers; - Spinlock m_command_lock; - DeprecatedWaitQueue m_command_completion_queue; + Mutex m_command_mutex; + SpinlockProtected m_current_command_complete { false }; + WaitQueue m_command_completion_queue; TransferRequestBlock m_command_result_transfer_request_block {}; u32 m_command_ring_enqueue_index { 0 }; u8 m_command_ring_producer_cycle_state { 1 }; diff --git a/Kernel/Bus/VirtIO/Transport/PCIe/TransportLink.cpp b/Kernel/Bus/VirtIO/Transport/PCIe/TransportLink.cpp index 4864c1fb3ec9b2..3b4faa6d5f2bb4 100644 --- a/Kernel/Bus/VirtIO/Transport/PCIe/TransportLink.cpp +++ b/Kernel/Bus/VirtIO/Transport/PCIe/TransportLink.cpp @@ -79,8 +79,7 @@ ErrorOr PCIeTransportLink::locate_configurations_and_resources(Badge PCIeTransportLink::locate_configurations_and_resources(Badge m_keymap_data {}; - size_t m_mouse_minor_number { 0 }; - size_t m_keyboard_minor_number { 0 }; + Atomic m_mouse_minor_number { 0 }; + Atomic m_keyboard_minor_number { 0 }; KeyboardClient* m_client { nullptr }; SpinlockProtected, LockRank::None> m_input_serial_io_controllers; diff --git a/Kernel/Devices/Input/VirtIO/Input.cpp b/Kernel/Devices/Input/VirtIO/Input.cpp index 0759e9b13fc52c..69028c796581ea 100644 --- a/Kernel/Devices/Input/VirtIO/Input.cpp +++ b/Kernel/Devices/Input/VirtIO/Input.cpp @@ -5,7 +5,6 @@ */ #include -#include #include #include #include diff --git a/Kernel/Devices/Serial/16550/PCISerial16550.cpp b/Kernel/Devices/Serial/16550/PCISerial16550.cpp index 5c13ec02816304..e5b67c19b5f24a 100644 --- a/Kernel/Devices/Serial/16550/PCISerial16550.cpp +++ b/Kernel/Devices/Serial/16550/PCISerial16550.cpp @@ -11,6 +11,7 @@ #include #include #include +#include namespace Kernel { @@ -65,30 +66,40 @@ ErrorOr Serial16550Driver::probe(PCI::DeviceIdentifier const& pci_device_i || pci_device_identifier.subclass_code() != PCI::SimpleCommunication::SubclassID::SerialController) return ENOTSUP; - auto initialize_serial_device = [&pci_device_identifier](BoardDefinition const& board_definition) { - auto registers_io_window = IOWindow::create_for_pci_device_bar(pci_device_identifier, static_cast(board_definition.pci_bar)).release_value_but_fixme_should_propagate_errors(); - auto first_offset_registers_io_window = registers_io_window->create_from_io_window_with_offset(board_definition.first_offset).release_value_but_fixme_should_propagate_errors(); + auto initialize_serial_device = [&pci_device_identifier](BoardDefinition const& board_definition) -> ErrorOr { + PCI::enable_io_space(pci_device_identifier); + + auto registers_io_window = TRY(IOWindow::create_for_pci_device_bar(pci_device_identifier, static_cast(board_definition.pci_bar))); + auto first_offset_registers_io_window = TRY(registers_io_window->create_from_io_window_with_offset(board_definition.first_offset)); for (size_t i = 0; i < board_definition.port_count; i++) { - auto port_registers_io_window = first_offset_registers_io_window->create_from_io_window_with_offset(board_definition.port_size * i).release_value_but_fixme_should_propagate_errors(); - auto serial_device = Device::try_create_device(move(port_registers_io_window), s_current_device_minor++).release_value_but_fixme_should_propagate_errors(); + auto port_registers_io_window = TRY(first_offset_registers_io_window->create_from_io_window_with_offset(board_definition.port_size * i)); + + auto serial_device = TRY(Device::try_create_device(move(port_registers_io_window), s_current_device_minor)); + s_current_device_minor++; + if (board_definition.baud_rate != Serial16550::Baud::Baud38400) // non-default baud serial_device->set_baud(board_definition.baud_rate); - // If this is the first port of the first pci serial device, store it as the debug PCI serial port (TODO: Make this configurable somehow?) - if (s_the == nullptr) - s_the = &serial_device.leak_ref(); // NOTE: We intentionally leak the reference to serial_device here. + if (is_pci_serial_debug_enabled()) { + // If this is the first port of the first pci serial device, store it as the debug PCI serial port (TODO: Make this configurable somehow?) + if (s_the == nullptr) + s_the = serial_device; + } + + (void)serial_device.leak_ref(); // NOTE: We intentionally leak the reference to serial_device here. } dmesgln("PCISerial16550: Found {} @ {}", board_definition.name, pci_device_identifier.address()); + + return {}; }; for (auto const& board_definition : board_definitions) { if (board_definition.device_id != pci_device_identifier.hardware_id()) continue; - initialize_serial_device(board_definition); - return {}; + return initialize_serial_device(board_definition); } // If we don't have a special board definition for this device and it's 16550-compatible, use a generic board definition. @@ -98,8 +109,7 @@ ErrorOr Serial16550Driver::probe(PCI::DeviceIdentifier const& pci_device_i PCI::SimpleCommunication::SerialControllerProgIf::CompatbileWith16750, PCI::SimpleCommunication::SerialControllerProgIf::CompatbileWith16850, PCI::SimpleCommunication::SerialControllerProgIf::CompatbileWith16950)) { - initialize_serial_device(generic_board_definition); - return {}; + return initialize_serial_device(generic_board_definition); } return ENOTSUP; diff --git a/Kernel/Devices/Serial/VirtIO/Console.cpp b/Kernel/Devices/Serial/VirtIO/Console.cpp index 97287936f7a77e..8cd01046daa8f9 100644 --- a/Kernel/Devices/Serial/VirtIO/Console.cpp +++ b/Kernel/Devices/Serial/VirtIO/Console.cpp @@ -44,7 +44,7 @@ UNMAP_AFTER_INIT ErrorOr Console::initialize_virtio_resources() } if (is_feature_accepted(VIRTIO_CONSOLE_F_MULTIPORT)) { max_nr_ports = transport_entity().config_read32(*cfg, 0x4); - m_ports.resize(max_nr_ports); + m_ports.try_resize(max_nr_ports).release_value_but_fixme_should_propagate_errors(); } }); dbgln("VirtIO::Console: cols: {}, rows: {}, max nr ports {}", cols, rows, max_nr_ports); @@ -57,7 +57,7 @@ UNMAP_AFTER_INIT ErrorOr Console::initialize_virtio_resources() } else { auto port = TRY(VirtIO::ConsolePort::create(0u, *this)); port->init_receive_buffer({}); - m_ports.append(port); + TRY(m_ports.try_append(port)); } return {}; } diff --git a/Kernel/Devices/Storage/AHCI/Controller.cpp b/Kernel/Devices/Storage/AHCI/Controller.cpp index 2e457a3b81ab31..92e9ff0c7cee05 100644 --- a/Kernel/Devices/Storage/AHCI/Controller.cpp +++ b/Kernel/Devices/Storage/AHCI/Controller.cpp @@ -223,7 +223,7 @@ LockRefPtr AHCIController::device(u32 index) const bit = bit_scan_forward(pi); if (checked_device.is_null()) continue; - connected_devices.append(checked_device.release_nonnull()); + connected_devices.try_append(checked_device.release_nonnull()).release_value_but_fixme_should_propagate_errors(); } dbgln_if(AHCI_DEBUG, "Connected device count: {}, Index: {}", connected_devices.size(), index); if (index >= connected_devices.size()) diff --git a/Kernel/Devices/Storage/AHCI/Definitions.h b/Kernel/Devices/Storage/AHCI/Definitions.h index 59b85cd739df3a..8b08e3af3d1791 100644 --- a/Kernel/Devices/Storage/AHCI/Definitions.h +++ b/Kernel/Devices/Storage/AHCI/Definitions.h @@ -181,7 +181,7 @@ class MaskedBitField { u32 bitfield = m_bitfield & m_bit_mask; for (size_t index = 0; index < 32; index++) { if (bitfield & 1) { - indices.append(index); + indices.try_append(index).release_value_but_fixme_should_propagate_errors(); } bitfield >>= 1; } diff --git a/Kernel/Devices/Storage/AHCI/Port.cpp b/Kernel/Devices/Storage/AHCI/Port.cpp index ab133f2c165434..8670d2090ee829 100644 --- a/Kernel/Devices/Storage/AHCI/Port.cpp +++ b/Kernel/Devices/Storage/AHCI/Port.cpp @@ -40,11 +40,11 @@ ErrorOr AHCIPort::allocate_resources_and_initialize_ports() for (size_t index = 0; index < 1; index++) { auto dma_page = TRY(MM.allocate_physical_page()); - m_dma_buffers.append(move(dma_page)); + TRY(m_dma_buffers.try_append(move(dma_page))); } for (size_t index = 0; index < 1; index++) { auto command_table_page = TRY(MM.allocate_physical_page()); - m_command_table_pages.append(move(command_table_page)); + TRY(m_command_table_pages.try_append(move(command_table_page))); } // FIXME: Synchronize DMA buffer accesses correctly and set the MemoryType to NonCacheable. @@ -417,7 +417,7 @@ Optional AHCIPort::prepare_and_set_scatter_li Vector> allocated_dma_regions; for (size_t index = 0; index < calculate_descriptors_count(request.block_count()); index++) { - allocated_dma_regions.append(m_dma_buffers.at(index)); + allocated_dma_regions.try_append(m_dma_buffers.at(index)).release_value_but_fixme_should_propagate_errors(); } m_current_scatter_list = Memory::ScatterGatherList::try_create(request, allocated_dma_regions.span(), m_connected_device->block_size(), "AHCI Scattered DMA"sv).release_value_but_fixme_should_propagate_errors(); diff --git a/Kernel/Devices/Storage/NVMe/NVMeController.cpp b/Kernel/Devices/Storage/NVMe/NVMeController.cpp index aa06f62a5de50b..e0265b428d09c2 100644 --- a/Kernel/Devices/Storage/NVMe/NVMeController.cpp +++ b/Kernel/Devices/Storage/NVMe/NVMeController.cpp @@ -211,7 +211,7 @@ UNMAP_AFTER_INIT ErrorOr NVMeController::identify_and_init_namespaces() dbgln_if(NVME_DEBUG, "NVMe: Block count is {} and Block size is {}", block_counts, block_size); - m_namespaces.append(TRY(NVMeNameSpace::create(*this, m_queues, nsid, block_counts, block_size))); + TRY(m_namespaces.try_append(TRY(NVMeNameSpace::create(*this, m_queues.clone().release_value_but_fixme_should_propagate_errors(), nsid, block_counts, block_size)))); m_device_count++; dbgln_if(NVME_DEBUG, "NVMe: Initialized namespace with NSID: {}", nsid); } @@ -443,7 +443,7 @@ UNMAP_AFTER_INIT ErrorOr NVMeController::create_io_queue(u8 qid, QueueType auto irq = TRY(allocate_irq(qid)); - m_queues.append(TRY(NVMeQueue::try_create(*this, qid, irq, IO_QUEUE_SIZE, move(cq_dma_region), move(sq_dma_region), move(doorbell), queue_type))); + TRY(m_queues.try_append(TRY(NVMeQueue::try_create(*this, qid, irq, IO_QUEUE_SIZE, move(cq_dma_region), move(sq_dma_region), move(doorbell), queue_type)))); dbgln_if(NVME_DEBUG, "NVMe: Created IO Queue with QID{}", m_queues.size()); return {}; } diff --git a/Kernel/Devices/Storage/StorageManagement.cpp b/Kernel/Devices/Storage/StorageManagement.cpp index 2fca880e2501c5..caf7e1d7a57df5 100644 --- a/Kernel/Devices/Storage/StorageManagement.cpp +++ b/Kernel/Devices/Storage/StorageManagement.cpp @@ -118,7 +118,7 @@ UNMAP_AFTER_INIT void StorageManagement::enumerate_pci_controllers(bool nvme_pol if (subclass_code == SubclassID::SATAController && device_identifier.prog_if() == PCI::MassStorage::SATAProgIF::AHCI) { if (auto ahci_controller_or_error = AHCIController::initialize(device_identifier); !ahci_controller_or_error.is_error()) - m_controllers.append(ahci_controller_or_error.value()); + m_controllers.try_append(ahci_controller_or_error.value()).release_value_but_fixme_should_propagate_errors(); else dmesgln("Unable to initialize AHCI controller: {}", ahci_controller_or_error.error()); } @@ -127,13 +127,13 @@ UNMAP_AFTER_INIT void StorageManagement::enumerate_pci_controllers(bool nvme_pol if (controller.is_error()) { dmesgln("Unable to initialize NVMe controller: {}", controller.error()); } else { - m_controllers.append(controller.release_value()); + m_controllers.try_append(controller.release_value()).release_value_but_fixme_should_propagate_errors(); } } if (VirtIOBlockController::is_handled(device_identifier)) { if (virtio_controller.is_null()) { auto controller = make_ref_counted(); - m_controllers.append(controller); + m_controllers.try_append(controller).release_value_but_fixme_should_propagate_errors(); virtio_controller = controller; } if (auto res = virtio_controller->add_device(device_identifier); res.is_error()) { @@ -152,7 +152,7 @@ UNMAP_AFTER_INIT void StorageManagement::enumerate_pci_controllers(bool nvme_pol if (sdhc_or_error.is_error()) { dmesgln("PCI: Failed to initialize SD Host Controller ({} - {}): {}", device_identifier.address(), device_identifier.hardware_id(), sdhc_or_error.error()); } else { - m_controllers.append(sdhc_or_error.release_value()); + m_controllers.try_append(sdhc_or_error.release_value()).release_value_but_fixme_should_propagate_errors(); } } }; @@ -470,13 +470,19 @@ ErrorOr> StorageManagement::create_first_vfs_root_ // NOTE: Fake a mounted count of 1 so the called VirtualFileSystem function in the // next pivot_root logic block thinks everything is OK. - fs->mounted_count().with([](auto& mounted_count) { + fs->mounted_count().with_exclusive([](auto& mounted_count) { mounted_count++; }); + // Adding filesystems to the all_file_systems_list usually happens in add_to_mounts_list_and_increment_fs_mounted_count(), + // but that function isn't called for the root filesystem of the first VFSRootContext. + FileSystem::all_file_systems_list().with([&fs](auto& fs_list) { + fs_list.append(*fs); + }); + TRY(VirtualFileSystem::pivot_root_by_copying_mounted_fs_instance(*vfs_root_context, *fs, root_mount_flags)); // NOTE: Return the mounted count to normal now we have it really mounted. - fs->mounted_count().with([](auto& mounted_count) { + fs->mounted_count().with_exclusive([](auto& mounted_count) { mounted_count--; }); return vfs_root_context; diff --git a/Kernel/Devices/Storage/StorageManagement.h b/Kernel/Devices/Storage/StorageManagement.h index 6fc0c3ff41f800..13d79cd05c527e 100644 --- a/Kernel/Devices/Storage/StorageManagement.h +++ b/Kernel/Devices/Storage/StorageManagement.h @@ -14,7 +14,6 @@ #include #include #include -#include #include #include diff --git a/Kernel/Devices/Storage/VirtIO/VirtIOBlockController.cpp b/Kernel/Devices/Storage/VirtIO/VirtIOBlockController.cpp index 5b24f82a7a2f1b..f3d5a3bfab54d1 100644 --- a/Kernel/Devices/Storage/VirtIO/VirtIOBlockController.cpp +++ b/Kernel/Devices/Storage/VirtIO/VirtIOBlockController.cpp @@ -36,7 +36,7 @@ ErrorOr VirtIOBlockController::add_device(PCI::DeviceIdentifier const& dev auto device = TRY(adopt_nonnull_lock_ref_or_enomem(new (nothrow) VirtIOBlockDevice(move(transport_link), lun, cid))); TRY(device->initialize_virtio_resources()); - m_devices.append(device); + TRY(m_devices.try_append(device)); return {}; } diff --git a/Kernel/Devices/TTY/PTYMultiplexer.cpp b/Kernel/Devices/TTY/PTYMultiplexer.cpp index d037cb2c65e404..b1266e13a3f234 100644 --- a/Kernel/Devices/TTY/PTYMultiplexer.cpp +++ b/Kernel/Devices/TTY/PTYMultiplexer.cpp @@ -26,7 +26,7 @@ UNMAP_AFTER_INIT PTYMultiplexer::PTYMultiplexer() : CharacterDevice(MajorAllocation::CharacterDeviceFamily::Console, 2) { m_freelist.with([&](auto& freelist) { - freelist.ensure_capacity(max_pty_pairs); + freelist.try_ensure_capacity(max_pty_pairs).release_value_but_fixme_should_propagate_errors(); for (int i = max_pty_pairs; i > 0; --i) freelist.unchecked_append(i - 1); }); @@ -58,7 +58,7 @@ ErrorOr> PTYMultiplexer::open(int options) void PTYMultiplexer::notify_master_destroyed(Badge, unsigned index) { m_freelist.with([&](auto& freelist) { - freelist.append(index); + freelist.try_append(index).release_value_but_fixme_should_propagate_errors(); dbgln_if(PTMX_DEBUG, "PTYMultiplexer: {} added to freelist", index); }); } diff --git a/Kernel/Devices/TTY/VirtualConsole.cpp b/Kernel/Devices/TTY/VirtualConsole.cpp index ceb21788c6308d..432ff9a60bf6da 100644 --- a/Kernel/Devices/TTY/VirtualConsole.cpp +++ b/Kernel/Devices/TTY/VirtualConsole.cpp @@ -155,7 +155,7 @@ void ConsoleImpl::set_size(u16 determined_columns, u16 determined_rows) m_normal_saved_state.cursor.clamp(rows() - 1, columns() - 1); m_alternate_saved_state.cursor.clamp(rows() - 1, columns() - 1); m_saved_cursor_position.clamp(rows() - 1, columns() - 1); - m_horizontal_tabs.resize(determined_columns); + m_horizontal_tabs.try_resize(determined_columns).release_value_but_fixme_should_propagate_errors(); for (unsigned i = 0; i < determined_columns; ++i) m_horizontal_tabs[i] = (i % 8) == 0; // Rightmost column is always last tab on line. @@ -238,7 +238,7 @@ UNMAP_AFTER_INIT void VirtualConsole::initialize() // Add the lines, so we also ensure they will be flushed now for (size_t row = 0; row < rows(); row++) { - m_lines.append({ true, 0 }); + m_lines.try_append({ true, 0 }).release_value_but_fixme_should_propagate_errors(); } VERIFY(m_cells); } @@ -259,7 +259,7 @@ void VirtualConsole::refresh_after_resolution_change() m_lines.shrink(rows()); } else { for (size_t row = 0; row < (size_t)(rows() - old_rows_count); row++) { - m_lines.append({ true, 0 }); + m_lines.try_append({ true, 0 }).release_value_but_fixme_should_propagate_errors(); } } diff --git a/Kernel/FileSystem/FATFS/Inode.cpp b/Kernel/FileSystem/FATFS/Inode.cpp index a1b285c11a319f..c37ef12fbe41bc 100644 --- a/Kernel/FileSystem/FATFS/Inode.cpp +++ b/Kernel/FileSystem/FATFS/Inode.cpp @@ -108,7 +108,7 @@ ErrorOr> FATInode::collect_sfns() ErrorOr FATInode::create_unique_sfn_for(FATEntry& entry, NonnullRefPtr sfn, Vector existing_sfns) { - auto is_sfn_unique = [existing_sfns](SFNUtils::SFN const& sfn) -> ErrorOr { + auto is_sfn_unique = [&existing_sfns](SFNUtils::SFN const& sfn) -> ErrorOr { auto serialized_name = TRY(sfn.serialize_name()); auto serialized_extension = TRY(sfn.serialize_extension()); for (auto const& current_sfn : existing_sfns) { @@ -386,7 +386,7 @@ ErrorOr FATInode::allocate_and_add_cluster_to_chain() TRY(fs().fat_write(cluster_list->last(), allocated_cluster)); } - cluster_list->append(allocated_cluster); + cluster_list->try_append(allocated_cluster).release_value_but_fixme_should_propagate_errors(); return {}; } diff --git a/Kernel/FileSystem/File.h b/Kernel/FileSystem/File.h index aa242f19531aad..cab196e45632fe 100644 --- a/Kernel/FileSystem/File.h +++ b/Kernel/FileSystem/File.h @@ -133,7 +133,7 @@ class File void evaluate_block_conditions() { - if (Processor::current_in_irq() != 0) { + if (Processor::current_in_irq()) { // If called from an IRQ handler we need to delay evaluation // and unblocking of waiting threads. Note that this File // instance may be deleted until the deferred call is executed! diff --git a/Kernel/FileSystem/FileSystem.cpp b/Kernel/FileSystem/FileSystem.cpp index 2ef4d5eabc6e83..ced53bb71d4a42 100644 --- a/Kernel/FileSystem/FileSystem.cpp +++ b/Kernel/FileSystem/FileSystem.cpp @@ -29,7 +29,7 @@ FileSystem::~FileSystem() ErrorOr FileSystem::prepare_to_unmount(Inode& mount_guest_inode) { - return m_attach_count.with([&](auto& attach_count) -> ErrorOr { + return m_attach_count.with_exclusive([&](auto& attach_count) -> ErrorOr { dbgln_if(VFS_DEBUG, "VFS: File system {} (id {}) is attached {} time(s)", class_name(), m_fsid.value(), attach_count); if (attach_count == 1) return prepare_to_clear_last_mount(mount_guest_inode); diff --git a/Kernel/FileSystem/FileSystem.h b/Kernel/FileSystem/FileSystem.h index b8d424e5eec67c..11a2d24545adcf 100644 --- a/Kernel/FileSystem/FileSystem.h +++ b/Kernel/FileSystem/FileSystem.h @@ -14,6 +14,7 @@ #include #include #include +#include #include namespace Kernel { @@ -66,7 +67,7 @@ class FileSystem : public AtomicRefCounted { // Converts file types that are used internally by the filesystem to DT_* types virtual u8 internal_file_type_to_directory_entry_type(DirectoryEntryView const& entry) const { return entry.file_type; } - SpinlockProtected& mounted_count() { return m_attach_count; } + MutexProtected& mounted_count() { return m_attach_count; } protected: FileSystem(); @@ -84,7 +85,7 @@ class FileSystem : public AtomicRefCounted { size_t m_fragment_size { 0 }; bool m_readonly { false }; - SpinlockProtected m_attach_count { 0 }; + MutexProtected m_attach_count { 0 }; IntrusiveListNode m_file_system_node; public: diff --git a/Kernel/FileSystem/Inode.cpp b/Kernel/FileSystem/Inode.cpp index 3f2d8d7a71043e..4cb24037780fbe 100644 --- a/Kernel/FileSystem/Inode.cpp +++ b/Kernel/FileSystem/Inode.cpp @@ -35,7 +35,7 @@ void Inode::sync_all() Inode::all_instances().with([&](auto& all_inodes) { for (auto& inode : all_inodes) { if (inode.is_metadata_dirty()) - inodes.append(inode); + inodes.try_append(inode).release_value_but_fixme_should_propagate_errors(); } }); diff --git a/Kernel/FileSystem/SysFS/Subsystems/Kernel/Configuration/CoredumpDirectory.cpp b/Kernel/FileSystem/SysFS/Subsystems/Kernel/Configuration/CoredumpDirectory.cpp index 0e84049eccda7e..8aaa38e8bd8836 100644 --- a/Kernel/FileSystem/SysFS/Subsystems/Kernel/Configuration/CoredumpDirectory.cpp +++ b/Kernel/FileSystem/SysFS/Subsystems/Kernel/Configuration/CoredumpDirectory.cpp @@ -28,11 +28,16 @@ ErrorOr> SysFSCoredumpDirectory::value() const return KString::try_create(""sv); }); } -void SysFSCoredumpDirectory::set_value(NonnullOwnPtr new_value) + +ErrorOr SysFSCoredumpDirectory::set_value(NonnullOwnPtr new_value) { + if (new_value->length() > 0 && new_value->bytes()[0] != '/') + return Error::from_errno(EINVAL); + Coredump::directory_path().with([&](auto& coredump_directory_path) { coredump_directory_path = move(new_value); }); + return {}; } mode_t SysFSCoredumpDirectory::permissions() const diff --git a/Kernel/FileSystem/SysFS/Subsystems/Kernel/Configuration/CoredumpDirectory.h b/Kernel/FileSystem/SysFS/Subsystems/Kernel/Configuration/CoredumpDirectory.h index fee5f43f10e363..e6e3a9b16a4321 100644 --- a/Kernel/FileSystem/SysFS/Subsystems/Kernel/Configuration/CoredumpDirectory.h +++ b/Kernel/FileSystem/SysFS/Subsystems/Kernel/Configuration/CoredumpDirectory.h @@ -20,7 +20,7 @@ class SysFSCoredumpDirectory final : public SysFSSystemStringVariable { private: virtual ErrorOr> value() const override; - virtual void set_value(NonnullOwnPtr new_value) override; + virtual ErrorOr set_value(NonnullOwnPtr new_value) override; explicit SysFSCoredumpDirectory(SysFSDirectory const&); diff --git a/Kernel/FileSystem/SysFS/Subsystems/Kernel/Configuration/StringVariable.cpp b/Kernel/FileSystem/SysFS/Subsystems/Kernel/Configuration/StringVariable.cpp index 5db643398b46e3..eb4a3621adb9aa 100644 --- a/Kernel/FileSystem/SysFS/Subsystems/Kernel/Configuration/StringVariable.cpp +++ b/Kernel/FileSystem/SysFS/Subsystems/Kernel/Configuration/StringVariable.cpp @@ -28,7 +28,7 @@ ErrorOr SysFSSystemStringVariable::write_bytes(off_t, size_t count, User // NOTE: If we are in a jail, don't let the current process to change the variable. if (Process::current().is_jailed()) return Error::from_errno(EPERM); - set_value(move(new_value_without_possible_newlines)); + TRY(set_value(move(new_value_without_possible_newlines))); return count; } diff --git a/Kernel/FileSystem/SysFS/Subsystems/Kernel/Configuration/StringVariable.h b/Kernel/FileSystem/SysFS/Subsystems/Kernel/Configuration/StringVariable.h index d02a2accba5dcb..da836a22cbb8fa 100644 --- a/Kernel/FileSystem/SysFS/Subsystems/Kernel/Configuration/StringVariable.h +++ b/Kernel/FileSystem/SysFS/Subsystems/Kernel/Configuration/StringVariable.h @@ -31,7 +31,7 @@ class SysFSSystemStringVariable : public SysFSGlobalInformation { { } virtual ErrorOr> value() const = 0; - virtual void set_value(NonnullOwnPtr new_value) = 0; + virtual ErrorOr set_value(NonnullOwnPtr new_value) = 0; private: // ^SysFSGlobalInformation diff --git a/Kernel/FileSystem/VFSRootContext.cpp b/Kernel/FileSystem/VFSRootContext.cpp index 6dbd484f63a373..0f13d7cae17a97 100644 --- a/Kernel/FileSystem/VFSRootContext.cpp +++ b/Kernel/FileSystem/VFSRootContext.cpp @@ -50,7 +50,7 @@ Custody const& VFSRootContext::empty_context_custody_for_kernel_processes() ErrorOr VFSRootContext::for_each_mount(Function(Mount const&)> callback) const { - return m_details.with([&](auto& details) -> ErrorOr { + return m_details.with_exclusive([&](auto& details) -> ErrorOr { for (auto& mount : details.mounts) TRY(callback(mount)); return {}; @@ -59,7 +59,7 @@ ErrorOr VFSRootContext::for_each_mount(Function(Mount const& void VFSRootContext::add_to_mounts_list_and_increment_fs_mounted_count(DoBindMount do_bind_mount, IntrusiveList<&Mount::m_vfs_list_node>& mounts_list, NonnullOwnPtr new_mount) { - new_mount->guest_fs().mounted_count().with([&](auto& mounted_count) { + new_mount->guest_fs().mounted_count().with_exclusive([&](auto& mounted_count) { // NOTE: We increment the mounted counter for the given filesystem regardless of the mount type, // as a bind mount also counts as a normal mount from the perspective of unmount(), // so we need to keep track of it in order for prepare_to_clear_last_mount() to work properly. @@ -98,7 +98,7 @@ ErrorOr> VFSRootContext::create_with_empty_ramfs() auto root_custody = TRY(Custody::try_create(nullptr, ""sv, fs->root_inode(), 0)); auto context = TRY(adopt_nonnull_ref_or_enomem(new (nothrow) VFSRootContext(root_custody))); auto new_mount = TRY(adopt_nonnull_own_or_enomem(new (nothrow) Mount(fs->root_inode(), 0))); - TRY(context->m_details.with([&](auto& details) -> ErrorOr { + TRY(context->m_details.with_exclusive([&](auto& details) -> ErrorOr { dbgln("VFSRootContext({}): Root (\"/\") FileSystemID {}, Mounting {} at inode {} with flags {}", context->id(), fs->fsid(), @@ -118,8 +118,8 @@ ErrorOr> VFSRootContext::create_with_empty_ramfs() ErrorOr VFSRootContext::pivot_root(FileBackedFileSystem::List& file_backed_file_systems_list, FileSystem& fs, NonnullOwnPtr new_mount, NonnullRefPtr root_mount_point, int root_mount_flags) { - return m_details.with([&](auto& details) -> ErrorOr { - return fs.mounted_count().with([&](auto& mounted_count) -> ErrorOr { + return m_details.with_exclusive([&](auto& details) -> ErrorOr { + return fs.mounted_count().with_exclusive([&](auto& mounted_count) -> ErrorOr { // NOTE: If the mounted count is 0, then this filesystem is about to be // deleted, so this must be a kernel bug as we don't include such filesystem // in the VirtualFileSystem s_details->file_backed_file_systems_list list anymore. @@ -175,7 +175,7 @@ ErrorOr VFSRootContext::do_full_teardown(Badge) while (unmount_was_successful) { unmount_was_successful = false; Vector mounts; - TRY(m_details.with([&mounts](auto& details) -> ErrorOr { + TRY(m_details.with_exclusive([&mounts](auto& details) -> ErrorOr { for (auto& mount : details.mounts) { TRY(mounts.try_append(const_cast(mount))); } @@ -208,7 +208,7 @@ ErrorOr VFSRootContext::do_full_teardown(Badge) ErrorOr VFSRootContext::unmount(FileBackedFileSystem::List& file_backed_file_systems_list, Inode& guest_inode, StringView custody_path) { - return m_details.with([&](auto& details) -> ErrorOr { + return m_details.with_exclusive([&](auto& details) -> ErrorOr { bool did_unmount = false; for (auto& mount : details.mounts) { if (&mount.guest() != &guest_inode) @@ -247,7 +247,7 @@ ErrorOr VFSRootContext::unmount(FileBackedFileSystem::List& file_backed_fi void VFSRootContext::detach(Badge) { - m_details.with([](auto& details) { + m_details.with_exclusive([](auto& details) { VERIFY(details.attached_by_process.was_set()); VERIFY(details.attach_count > 0); details.attach_count--; @@ -256,7 +256,7 @@ void VFSRootContext::detach(Badge) void VFSRootContext::attach(Badge) { - m_details.with([](auto& details) { + m_details.with_exclusive([](auto& details) { details.attached_by_process.set(); details.attach_count++; }); @@ -272,7 +272,7 @@ bool VFSRootContext::mount_point_exists_at_custody(Custody& mount_point, Details ErrorOr VFSRootContext::do_on_mount_for_host_custody(ValidateImmutableFlag validate_immutable_flag, Custody const& current_custody, Function callback) const { VERIFY(validate_immutable_flag == ValidateImmutableFlag::Yes || validate_immutable_flag == ValidateImmutableFlag::No); - return m_details.with([&](auto& details) -> ErrorOr { + return m_details.with_exclusive([&](auto& details) -> ErrorOr { // NOTE: We either search for the root mount or for a mount that has a parent custody! if (!current_custody.parent()) { for (auto& mount : details.mounts) { @@ -326,7 +326,7 @@ ErrorOr VFSRootContext::current_mount_state_f ErrorOr VFSRootContext::add_new_mount(DoBindMount do_bind_mount, Inode& source, Custody& mount_point, int flags) { auto new_mount = TRY(adopt_nonnull_own_or_enomem(new (nothrow) Mount(source, mount_point, flags))); - return m_details.with([&](auto& details) -> ErrorOr { + return m_details.with_exclusive([&](auto& details) -> ErrorOr { // NOTE: The VFSRootContext should be attached to the list if there's // at least one mount in the mount table. // We also should have at least one mount in the table because diff --git a/Kernel/FileSystem/VFSRootContext.h b/Kernel/FileSystem/VFSRootContext.h index 1301188459d46b..adbb9927d9879f 100644 --- a/Kernel/FileSystem/VFSRootContext.h +++ b/Kernel/FileSystem/VFSRootContext.h @@ -88,7 +88,7 @@ class VFSRootContext : public AtomicRefCounted { return {}; } - mutable SpinlockProtected m_details {}; + mutable MutexProtected
m_details {}; SpinlockProtected, LockRank::None> m_root_custody; diff --git a/Kernel/FileSystem/VirtualFileSystem.cpp b/Kernel/FileSystem/VirtualFileSystem.cpp index bde41939e9f435..c45e49e1233c66 100644 --- a/Kernel/FileSystem/VirtualFileSystem.cpp +++ b/Kernel/FileSystem/VirtualFileSystem.cpp @@ -273,7 +273,7 @@ void VirtualFileSystem::sync_filesystems() Vector, 32> file_systems; s_details->file_systems_list.with([&](auto const& list) { for (auto& fs : list) - file_systems.append(fs); + file_systems.try_append(fs).release_value_but_fixme_should_propagate_errors(); }); for (auto& fs : file_systems) { @@ -295,7 +295,7 @@ ErrorOr VirtualFileSystem::remove_mount(Mount& mount, FileBackedFileSystem { NonnullRefPtr fs = mount.guest_fs(); TRY(fs->prepare_to_unmount(mount.guest())); - fs->mounted_count().with([&](auto& mounted_count) { + fs->mounted_count().with_exclusive([&](auto& mounted_count) { VERIFY(mounted_count > 0); if (mounted_count == 1) { dbgln("VirtualFileSystem: Unmounting file system {} for the last time...", fs->fsid()); diff --git a/Kernel/Firmware/ACPI/Parser.cpp b/Kernel/Firmware/ACPI/Parser.cpp index 2ed794113e6b45..71bff047119e17 100644 --- a/Kernel/Firmware/ACPI/Parser.cpp +++ b/Kernel/Firmware/ACPI/Parser.cpp @@ -77,7 +77,7 @@ UNMAP_AFTER_INIT void ACPISysFSDirectory::find_tables_and_register_them_as_compo MUST(m_child_components.with([&](auto& list) -> ErrorOr { ACPI::Parser::the()->enumerate_static_tables([&](StringView signature, PhysicalAddress p_table, size_t length) { if (signature == "SSDT") { - auto component_name = KString::formatted("{:4s}{}", signature.characters_without_null_termination(), ssdt_count).release_value_but_fixme_should_propagate_errors(); + auto component_name = KString::formatted("{:4s}{}", signature, ssdt_count).release_value_but_fixme_should_propagate_errors(); list.append(ACPISysFSComponent::create(component_name->view(), p_table, length)); ssdt_count++; return; @@ -207,7 +207,7 @@ UNMAP_AFTER_INIT void Parser::process_dsdt() auto sdt = Memory::map_typed(m_fadt).release_value_but_fixme_should_propagate_errors(); // Add DSDT-pointer to expose the full table in /sys/firmware/acpi/ - m_sdt_pointers.append(PhysicalAddress(sdt->dsdt_ptr)); + m_sdt_pointers.try_append(PhysicalAddress(sdt->dsdt_ptr)).release_value_but_fixme_should_propagate_errors(); auto dsdt_or_error = Memory::map_typed(PhysicalAddress(sdt->dsdt_ptr)); if (dsdt_or_error.is_error()) { @@ -361,7 +361,7 @@ UNMAP_AFTER_INIT void Parser::initialize_main_system_description_table() dbgln_if(ACPI_DEBUG, "ACPI: XSDT pointer @ {}", VirtualAddress { &xsdt }); for (u32 i = 0; i < ((length - sizeof(Structures::SDTHeader)) / sizeof(u64)); i++) { dbgln_if(ACPI_DEBUG, "ACPI: Found new table [{0}], @ V{1:p} - P{1:p}", i, &xsdt.table_ptrs[i]); - m_sdt_pointers.append(PhysicalAddress(xsdt.table_ptrs[i])); + m_sdt_pointers.try_append(PhysicalAddress(xsdt.table_ptrs[i])).release_value_but_fixme_should_propagate_errors(); } } else { auto& rsdt = (Structures::RSDT const&)*sdt; @@ -370,7 +370,7 @@ UNMAP_AFTER_INIT void Parser::initialize_main_system_description_table() dbgln_if(ACPI_DEBUG, "ACPI: RSDT pointer @ V{}", &rsdt); for (u32 i = 0; i < ((length - sizeof(Structures::SDTHeader)) / sizeof(u32)); i++) { dbgln_if(ACPI_DEBUG, "ACPI: Found new table [{0}], @ V{1:p} - P{1:p}", i, &rsdt.table_ptrs[i]); - m_sdt_pointers.append(PhysicalAddress(rsdt.table_ptrs[i])); + m_sdt_pointers.try_append(PhysicalAddress(rsdt.table_ptrs[i])).release_value_but_fixme_should_propagate_errors(); } } } diff --git a/Kernel/Firmware/ACPI/StaticParsing.cpp b/Kernel/Firmware/ACPI/StaticParsing.cpp index 3f01ce639eb36d..6c568efb90a0f2 100644 --- a/Kernel/Firmware/ACPI/StaticParsing.cpp +++ b/Kernel/Firmware/ACPI/StaticParsing.cpp @@ -4,6 +4,7 @@ * SPDX-License-Identifier: BSD-2-Clause */ +#include #include #include #include @@ -31,16 +32,33 @@ Optional find_rsdp() static bool match_table_signature(PhysicalAddress table_header, StringView signature) { - // FIXME: There's no validation of ACPI tables here. Use the checksum to validate the tables. VERIFY(signature.length() == 4); - auto table = Memory::map_typed(table_header).release_value_but_fixme_should_propagate_errors(); - return !strncmp(table->h.sig, signature.characters_without_null_termination(), 4); + auto check_in_region = [&](size_t length, auto&& callback) -> bool { + auto region = Memory::map_typed(table_header, length).release_value_but_fixme_should_propagate_errors(); + return callback(region.ptr()); + }; + + u32 length = 0; + auto check_signature_and_get_length = [&](u8 const* base) -> bool { + if (memcmp(reinterpret_cast(base), signature.characters_without_null_termination(), 4) != 0) + return false; + length = ByteReader::load32(base + 4); + return true; + }; + + auto validate_checksum = [&](u8 const* base) -> bool { + u8 checksum = 0; + for (u32 i = 0; i < length; ++i) + checksum += base[i]; + return checksum == 0; + }; + + return check_in_region(8, check_signature_and_get_length) && check_in_region(length, validate_checksum); } ErrorOr> search_table_in_xsdt(PhysicalAddress xsdt_address, StringView signature) { - // FIXME: There's no validation of ACPI tables here. Use the checksum to validate the tables. VERIFY(signature.length() == 4); auto xsdt = TRY(Memory::map_typed(xsdt_address)); @@ -54,7 +72,6 @@ ErrorOr> search_table_in_xsdt(PhysicalAddress xsdt_add ErrorOr> find_table(PhysicalAddress rsdp_address, StringView signature) { - // FIXME: There's no validation of ACPI tables here. Use the checksum to validate the tables. VERIFY(signature.length() == 4); auto rsdp = TRY(Memory::map_typed(rsdp_address)); @@ -72,7 +89,6 @@ ErrorOr> find_table(PhysicalAddress rsdp_address, Stri ErrorOr> search_table_in_rsdt(PhysicalAddress rsdt_address, StringView signature) { - // FIXME: There's no validation of ACPI tables here. Use the checksum to validate the tables. VERIFY(signature.length() == 4); auto rsdt = TRY(Memory::map_typed(rsdt_address)); diff --git a/Kernel/Firmware/DeviceTree/Device.cpp b/Kernel/Firmware/DeviceTree/Device.cpp index c5cab5daa0b3b0..ff8e88f1d1104e 100644 --- a/Kernel/Firmware/DeviceTree/Device.cpp +++ b/Kernel/Firmware/DeviceTree/Device.cpp @@ -5,6 +5,8 @@ */ #include +#include +#include namespace Kernel::DeviceTree { @@ -17,4 +19,14 @@ ErrorOr Device::get_resource(size_t index) const }; } +ErrorOr Device::get_interrupt_number(size_t index) const +{ + auto interrupts = TRY(node().interrupts(DeviceTree::get())); + auto maybe_interrupt = interrupts.get(index); + if (!maybe_interrupt.has_value()) + return EINVAL; + + return Management::the().resolve_interrupt_number(*maybe_interrupt); +} + } diff --git a/Kernel/Firmware/DeviceTree/Device.h b/Kernel/Firmware/DeviceTree/Device.h index 587b63ea8cc339..2fd267cdeec306 100644 --- a/Kernel/Firmware/DeviceTree/Device.h +++ b/Kernel/Firmware/DeviceTree/Device.h @@ -43,6 +43,9 @@ class Device { }; ErrorOr get_resource(size_t index) const; + // FIXME: Add support for the "interrupt-names" property to resolve interrupts by name. + ErrorOr get_interrupt_number(size_t index) const; + private: // This needs to be a pointer for the class to be movable. ::DeviceTree::Node const* m_node; diff --git a/Kernel/Firmware/DeviceTree/DeviceRecipe.h b/Kernel/Firmware/DeviceTree/DeviceRecipe.h deleted file mode 100644 index a5256e21dacd11..00000000000000 --- a/Kernel/Firmware/DeviceTree/DeviceRecipe.h +++ /dev/null @@ -1,20 +0,0 @@ -/* - * Copyright (c) 2024, Leon Albrecht - * - * SPDX-License-Identifier: BSD-2-Clause - */ - -#pragma once - -#include - -namespace Kernel::DeviceTree { - -template -struct DeviceRecipe { - StringView driver_name; - StringView node_name; - Function()> create_device; -}; - -} diff --git a/Kernel/Firmware/DeviceTree/Driver.h b/Kernel/Firmware/DeviceTree/Driver.h index 27a25981f052fc..1d9525313cb8e0 100644 --- a/Kernel/Firmware/DeviceTree/Driver.h +++ b/Kernel/Firmware/DeviceTree/Driver.h @@ -19,6 +19,7 @@ class Driver { public: enum class ProbeStage { + InterruptController, Early, Regular, }; @@ -65,5 +66,6 @@ class Driver { #define DEVICETREE_DRIVER(driver_name, compatibles_array) __DEVICETREE_DRIVER(driver_name, compatibles_array, Kernel::DeviceTree::Driver::ProbeStage::Regular) #define EARLY_DEVICETREE_DRIVER(driver_name, compatibles_array) __DEVICETREE_DRIVER(driver_name, compatibles_array, Kernel::DeviceTree::Driver::ProbeStage::Early) +#define INTERRUPT_CONTROLLER_DEVICETREE_DRIVER(driver_name, compatibles_array) __DEVICETREE_DRIVER(driver_name, compatibles_array, Kernel::DeviceTree::Driver::ProbeStage::InterruptController) } diff --git a/Kernel/Firmware/DeviceTree/InterruptController.h b/Kernel/Firmware/DeviceTree/InterruptController.h new file mode 100644 index 00000000000000..c30378271b1dc7 --- /dev/null +++ b/Kernel/Firmware/DeviceTree/InterruptController.h @@ -0,0 +1,20 @@ +/* + * Copyright (c) 2025, Sönke Holz + * + * SPDX-License-Identifier: BSD-2-Clause + */ + +#pragma once + +#include + +namespace Kernel::DeviceTree { + +class InterruptController { +public: + virtual ~InterruptController() = default; + + virtual ErrorOr translate_interrupt_specifier_to_interrupt_number(ReadonlyBytes) const = 0; +}; + +} diff --git a/Kernel/Firmware/DeviceTree/Management.cpp b/Kernel/Firmware/DeviceTree/Management.cpp index 42bea829480ec5..6a5c1191378cfc 100644 --- a/Kernel/Firmware/DeviceTree/Management.cpp +++ b/Kernel/Firmware/DeviceTree/Management.cpp @@ -19,8 +19,6 @@ void Management::initialize() dmesgln("DeviceTree: System board model: {}", maybe_model->as_string()); MUST(the().scan_node_for_devices(DeviceTree::get())); - - MUST(the().probe_drivers(Driver::ProbeStage::Early)); } Management& Management::the() @@ -41,6 +39,21 @@ ErrorOr Management::register_driver(NonnullOwnPtr&& dr return {}; } +ErrorOr Management::register_interrupt_controller(DeviceTree::Device const& device, DeviceTree::InterruptController const& interrupt_controller) +{ + TRY(the().m_interrupt_controllers.try_set(&device.node(), &interrupt_controller)); + return {}; +} + +ErrorOr Management::resolve_interrupt_number(::DeviceTree::Interrupt interrupt) const +{ + auto maybe_interrupt_controller = m_interrupt_controllers.get(interrupt.domain_root); + if (!maybe_interrupt_controller.has_value()) + return ENOENT; + + return maybe_interrupt_controller.value()->translate_interrupt_specifier_to_interrupt_number(interrupt.interrupt_specifier); +} + ErrorOr Management::scan_node_for_devices(::DeviceTree::Node const& node) { for (auto const& [child_name, child] : node.children()) { diff --git a/Kernel/Firmware/DeviceTree/Management.h b/Kernel/Firmware/DeviceTree/Management.h index 35163d07374d81..3988ac8f64e979 100644 --- a/Kernel/Firmware/DeviceTree/Management.h +++ b/Kernel/Firmware/DeviceTree/Management.h @@ -8,6 +8,7 @@ #include #include +#include #include namespace Kernel::DeviceTree { @@ -18,6 +19,9 @@ class Management { static Management& the(); static ErrorOr register_driver(NonnullOwnPtr&&); + static ErrorOr register_interrupt_controller(DeviceTree::Device const&, DeviceTree::InterruptController const&); + + ErrorOr resolve_interrupt_number(::DeviceTree::Interrupt) const; ErrorOr scan_node_for_devices(::DeviceTree::Node const& node); @@ -30,6 +34,8 @@ class Management { HashMap m_driver_by_compatible_string; HashMap<::DeviceTree::Node const*, Device> m_devices; + + HashMap<::DeviceTree::Node const*, DeviceTree::InterruptController const*> m_interrupt_controllers; }; } diff --git a/Kernel/Library/IOWindow.cpp b/Kernel/Library/IOWindow.cpp index de6d5beddf8ec6..da5063df4f42cc 100644 --- a/Kernel/Library/IOWindow.cpp +++ b/Kernel/Library/IOWindow.cpp @@ -85,8 +85,9 @@ ErrorOr> IOWindow::create_for_pci_device_bar(PCI::Device auto io_address_range = TRY(adopt_nonnull_own_or_enomem(new (nothrow) IOAddressData((pci_bar_value & 0xfffffffc), space_length))); return TRY(adopt_nonnull_own_or_enomem(new (nothrow) IOWindow(move(io_address_range)))); #else - // Note: For non-x86 platforms, IO PCI BARs are simply not useable. - return Error::from_errno(ENOTSUP); + // On non-x86 systems, I/O BARs are mapped to MMIO instead of using special I/O access instructions. + auto memory_mapped_range = TRY(PCI::adopt_new_nonnull_own_bar_mapping(pci_device_identifier, pci_bar, space_length)); + return TRY(adopt_nonnull_own_or_enomem(new (nothrow) IOWindow(move(memory_mapped_range)))); #endif } diff --git a/Kernel/Library/MiniStdLib.cpp b/Kernel/Library/MiniStdLib.cpp index 9f2ce74086a3fc..3f3e50fd222f08 100644 --- a/Kernel/Library/MiniStdLib.cpp +++ b/Kernel/Library/MiniStdLib.cpp @@ -70,10 +70,37 @@ void* memset(void* dest_ptr, int c, size_t n) : "a"(c) : "memory"); #else - u8* pd = (u8*)dest_ptr; - for (; n--;) - *pd++ = c; + auto* dest = static_cast(dest_ptr); + + auto is_word_aligned = [](auto ptr) { + return reinterpret_cast(ptr) % sizeof(FlatPtr) == 0; + }; + + // Set bytes until destination is word-aligned. + while (n > 0 && !is_word_aligned(dest)) { + *dest++ = static_cast(c); + n--; + } + + // Set in word-sized chunks. + FlatPtr exploded = explode_byte(c); + + auto* dest_word = reinterpret_cast(dest); + +# pragma GCC unroll 8 + while (n >= sizeof(FlatPtr)) { + *dest_word++ = exploded; + n -= sizeof(FlatPtr); + } + dest = reinterpret_cast(dest_word); + + // Set remaining tail bytes. + while (n > 0) { + *dest++ = static_cast(c); + n--; + } #endif + return dest_ptr; } diff --git a/Kernel/Memory/MappedROM.h b/Kernel/Memory/MappedROM.h index dfef6d1a116cf7..383799e3e1eb3c 100644 --- a/Kernel/Memory/MappedROM.h +++ b/Kernel/Memory/MappedROM.h @@ -21,16 +21,24 @@ class MappedROM { size_t offset { 0 }; PhysicalAddress paddr; - Optional find_chunk_starting_with(StringView prefix, size_t chunk_size) const + u8 const* pointer_to_chunk_starting_with(StringView prefix, size_t chunk_size) const { auto prefix_length = prefix.length(); if (size < prefix_length) - return {}; + return nullptr; for (auto* candidate = base(); candidate <= end() - prefix_length; candidate += chunk_size) { if (!__builtin_memcmp(prefix.characters_without_null_termination(), candidate, prefix.length())) - return paddr_of(candidate); + return candidate; } - return {}; + return nullptr; + } + + Optional find_chunk_starting_with(StringView prefix, size_t chunk_size) const + { + auto result = pointer_to_chunk_starting_with(prefix, chunk_size); + if (!result) + return {}; + return paddr_of(result); } PhysicalAddress paddr_of(u8 const* ptr) const { return paddr.offset(ptr - this->base()); } diff --git a/Kernel/Memory/MemoryManager.cpp b/Kernel/Memory/MemoryManager.cpp index 8c8dd5e0444ab5..18063c86da0905 100644 --- a/Kernel/Memory/MemoryManager.cpp +++ b/Kernel/Memory/MemoryManager.cpp @@ -216,7 +216,7 @@ UNMAP_AFTER_INIT void MemoryManager::register_reserved_ranges(GlobalData& global if (current_range.type != PhysicalMemoryRangeType::Reserved) { if (range.start.is_null()) continue; - global_data.reserved_memory_ranges.append(ContiguousReservedMemoryRange { range.start, current_range.start.get() - range.start.get() }); + global_data.reserved_memory_ranges.try_append(ContiguousReservedMemoryRange { range.start, current_range.start.get() - range.start.get() }).release_value_but_fixme_should_propagate_errors(); range.start.set((FlatPtr) nullptr); continue; } @@ -229,7 +229,7 @@ UNMAP_AFTER_INIT void MemoryManager::register_reserved_ranges(GlobalData& global return; if (range.start.is_null()) return; - global_data.reserved_memory_ranges.append(ContiguousReservedMemoryRange { range.start, global_data.physical_memory_ranges.last().start.get() + global_data.physical_memory_ranges.last().length - range.start.get() }); + global_data.reserved_memory_ranges.try_append(ContiguousReservedMemoryRange { range.start, global_data.physical_memory_ranges.last().start.get() + global_data.physical_memory_ranges.last().length - range.start.get() }).release_value_but_fixme_should_propagate_errors(); } bool MemoryManager::is_allowed_to_read_physical_memory_for_userspace(PhysicalAddress start_address, size_t read_length) const @@ -256,7 +256,7 @@ UNMAP_AFTER_INIT void MemoryManager::parse_memory_map() { // Register used memory regions that we know of. m_global_data.with([this](auto& global_data) { - global_data.used_memory_ranges.ensure_capacity(4); + global_data.used_memory_ranges.try_ensure_capacity(4).release_value_but_fixme_should_propagate_errors(); #if ARCH(X86_64) // NOTE: We don't touch the first 1 MiB of RAM on x86-64 even if it's usable as indicated // by a certain memory map. There are 2 reasons for this: @@ -280,9 +280,9 @@ UNMAP_AFTER_INIT void MemoryManager::parse_memory_map() // **To be completely on the safe side** and never worry about where the EBDA is located, how BIOS might // corrupt the low memory range during power state changing, other bad behavior of some BIOS might change // a value in the very first 64k bytes of RAM, etc - we should just ignore this range completely. - global_data.used_memory_ranges.append(UsedMemoryRange { UsedMemoryRangeType::LowMemory, PhysicalAddress(0x00000000), PhysicalAddress(1 * MiB) }); + global_data.used_memory_ranges.try_append(UsedMemoryRange { UsedMemoryRangeType::LowMemory, PhysicalAddress(0x00000000), PhysicalAddress(1 * MiB) }).release_value_but_fixme_should_propagate_errors(); #endif - global_data.used_memory_ranges.append(UsedMemoryRange { UsedMemoryRangeType::Kernel, PhysicalAddress(virtual_to_low_physical((FlatPtr)start_of_kernel_image)), PhysicalAddress(page_round_up(virtual_to_low_physical((FlatPtr)end_of_kernel_image)).release_value_but_fixme_should_propagate_errors()) }); + global_data.used_memory_ranges.try_append(UsedMemoryRange { UsedMemoryRangeType::Kernel, PhysicalAddress(virtual_to_low_physical((FlatPtr)start_of_kernel_image)), PhysicalAddress(page_round_up(virtual_to_low_physical((FlatPtr)end_of_kernel_image)).release_value_but_fixme_should_propagate_errors()) }).release_value_but_fixme_should_propagate_errors(); if (g_boot_info.boot_method == BootMethod::EFI) parse_memory_map_efi(global_data); @@ -343,7 +343,7 @@ UNMAP_AFTER_INIT void MemoryManager::parse_memory_map() if (last_contiguous_physical_range.has_value()) { auto range = last_contiguous_physical_range.release_value(); // FIXME: OOM? - global_data.physical_regions.append(PhysicalRegion::try_create(range.lower, range.upper).release_nonnull()); + global_data.physical_regions.try_append(PhysicalRegion::try_create(range.lower, range.upper).release_nonnull()).release_value_but_fixme_should_propagate_errors(); } last_contiguous_physical_range = ContiguousPhysicalRange { .lower = addr, .upper = addr }; } else { @@ -354,7 +354,7 @@ UNMAP_AFTER_INIT void MemoryManager::parse_memory_map() if (last_contiguous_physical_range.has_value()) { auto range = last_contiguous_physical_range.release_value(); // FIXME: OOM? - global_data.physical_regions.append(PhysicalRegion::try_create(range.lower, range.upper).release_nonnull()); + global_data.physical_regions.try_append(PhysicalRegion::try_create(range.lower, range.upper).release_nonnull()).release_value_but_fixme_should_propagate_errors(); } } @@ -435,7 +435,7 @@ UNMAP_AFTER_INIT void MemoryManager::parse_memory_map_efi(MemoryManager::GlobalD case EFI::MemoryType::BootServicesCode: case EFI::MemoryType::BootServicesData: case EFI::MemoryType::Conventional: - global_data.physical_memory_ranges.append(PhysicalMemoryRange { PhysicalMemoryRangeType::Usable, start_paddr, length }); + global_data.physical_memory_ranges.try_append(PhysicalMemoryRange { PhysicalMemoryRangeType::Usable, start_paddr, length }).release_value_but_fixme_should_propagate_errors(); break; case EFI::MemoryType::Reserved: case EFI::MemoryType::LoaderCode: @@ -466,29 +466,29 @@ UNMAP_AFTER_INIT void MemoryManager::parse_memory_map_efi(MemoryManager::GlobalD if ((start_paddr.get() != 0x000000fd00000000 || length != (0x000000ffffffffff - 0x000000fd00000000) + 1) && (start_paddr.get() != 0x000003fff0000000 || length != 0x10000000)) #endif - global_data.physical_memory_ranges.append(PhysicalMemoryRange { PhysicalMemoryRangeType::Reserved, start_paddr, length }); + global_data.physical_memory_ranges.try_append(PhysicalMemoryRange { PhysicalMemoryRangeType::Reserved, start_paddr, length }).release_value_but_fixme_should_propagate_errors(); break; case EFI::MemoryType::ACPIReclaim: - global_data.physical_memory_ranges.append(PhysicalMemoryRange { PhysicalMemoryRangeType::ACPI_Reclaimable, start_paddr, length }); + global_data.physical_memory_ranges.try_append(PhysicalMemoryRange { PhysicalMemoryRangeType::ACPI_Reclaimable, start_paddr, length }).release_value_but_fixme_should_propagate_errors(); break; case EFI::MemoryType::ACPI_NVS: - global_data.physical_memory_ranges.append(PhysicalMemoryRange { PhysicalMemoryRangeType::ACPI_NVS, start_paddr, length }); + global_data.physical_memory_ranges.try_append(PhysicalMemoryRange { PhysicalMemoryRangeType::ACPI_NVS, start_paddr, length }).release_value_but_fixme_should_propagate_errors(); break; case EFI::MemoryType::Unusable: dmesgln("MM: Warning, detected bad memory range!"); - global_data.physical_memory_ranges.append(PhysicalMemoryRange { PhysicalMemoryRangeType::BadMemory, start_paddr, length }); + global_data.physical_memory_ranges.try_append(PhysicalMemoryRange { PhysicalMemoryRangeType::BadMemory, start_paddr, length }).release_value_but_fixme_should_propagate_errors(); break; default: dbgln("MM: Unknown EFI memory type: {}", to_underlying(descriptor->type)); - global_data.physical_memory_ranges.append(PhysicalMemoryRange { PhysicalMemoryRangeType::Unknown, start_paddr, length }); + global_data.physical_memory_ranges.try_append(PhysicalMemoryRange { PhysicalMemoryRangeType::Unknown, start_paddr, length }).release_value_but_fixme_should_propagate_errors(); break; } } // SMBIOS data can be in a BootServicesData memory region (see https://uefi.org/specs/UEFI/2.10/02_Overview.html#x64-platforms, the same requirement is listed for AArch64 and RISC-V as well). // BootServices* memory regions are treated as normal main memory after ExitBootServices, so we need to explicitly mark its ranges as used. - global_data.used_memory_ranges.append(UsedMemoryRange { UsedMemoryRangeType::SMBIOS, g_boot_info.smbios.entry_point_paddr, g_boot_info.smbios.entry_point_paddr.offset(g_boot_info.smbios.entry_point_length) }); - global_data.used_memory_ranges.append(UsedMemoryRange { UsedMemoryRangeType::SMBIOS, g_boot_info.smbios.structure_table_paddr, g_boot_info.smbios.structure_table_paddr.offset(g_boot_info.smbios.maximum_structure_table_length) }); + global_data.used_memory_ranges.try_append(UsedMemoryRange { UsedMemoryRangeType::SMBIOS, g_boot_info.smbios.entry_point_paddr, g_boot_info.smbios.entry_point_paddr.offset(g_boot_info.smbios.entry_point_length) }).release_value_but_fixme_should_propagate_errors(); + global_data.used_memory_ranges.try_append(UsedMemoryRange { UsedMemoryRangeType::SMBIOS, g_boot_info.smbios.structure_table_paddr, g_boot_info.smbios.structure_table_paddr.offset(g_boot_info.smbios.maximum_structure_table_length) }).release_value_but_fixme_should_propagate_errors(); } UNMAP_AFTER_INIT void MemoryManager::parse_memory_map_fdt(MemoryManager::GlobalData& global_data, u8 const* fdt_addr) @@ -503,9 +503,9 @@ UNMAP_AFTER_INIT void MemoryManager::parse_memory_map_fdt(MemoryManager::GlobalD u64 next_block_offset = fdt_header.off_mem_rsvmap + sizeof(::DeviceTree::FlattenedDeviceTreeReserveEntry); while ((next_block_offset < fdt_header.off_dt_struct) && (*mem_reserve_block != ::DeviceTree::FlattenedDeviceTreeReserveEntry {})) { dbgln("MM: Reserved Range /memreserve/: address: {} size {:#x}", PhysicalAddress { mem_reserve_block->address }, mem_reserve_block->size); - global_data.physical_memory_ranges.append(PhysicalMemoryRange { PhysicalMemoryRangeType::Reserved, PhysicalAddress { mem_reserve_block->address }, mem_reserve_block->size }); + global_data.physical_memory_ranges.try_append(PhysicalMemoryRange { PhysicalMemoryRangeType::Reserved, PhysicalAddress { mem_reserve_block->address }, mem_reserve_block->size }).release_value_but_fixme_should_propagate_errors(); // FIXME: Not all of these are "used", only those in "memory" are actually "used" - global_data.used_memory_ranges.append(UsedMemoryRange { UsedMemoryRangeType::BootModule, PhysicalAddress { mem_reserve_block->address }, PhysicalAddress { mem_reserve_block->address + mem_reserve_block->size } }); + global_data.used_memory_ranges.try_append(UsedMemoryRange { UsedMemoryRangeType::BootModule, PhysicalAddress { mem_reserve_block->address }, PhysicalAddress { mem_reserve_block->address + mem_reserve_block->size } }).release_value_but_fixme_should_propagate_errors(); ++mem_reserve_block; next_block_offset += sizeof(::DeviceTree::FlattenedDeviceTreeReserveEntry); } @@ -574,7 +574,7 @@ UNMAP_AFTER_INIT void MemoryManager::parse_memory_map_fdt(MemoryManager::GlobalD state.state = State::Root; break; case State::InMemory: - global_data.physical_memory_ranges.grow_capacity(global_data.physical_memory_ranges.size() + state.reg.size()); + global_data.physical_memory_ranges.try_grow_capacity(global_data.physical_memory_ranges.size() + state.reg.size()).release_value_but_fixme_should_propagate_errors(); for (auto const& reg_entry : state.reg) { dbgln("MM: Memory Range {}: address: {} size {:#x}", node_name, PhysicalAddress { reg_entry.start_addr }, reg_entry.size); @@ -589,8 +589,8 @@ UNMAP_AFTER_INIT void MemoryManager::parse_memory_map_fdt(MemoryManager::GlobalD if (state.reg.is_empty()) dbgln("MM: Skipping dynamically allocated reserved memory region {}", node_name); - global_data.physical_memory_ranges.grow_capacity(global_data.physical_memory_ranges.size() + state.reg.size()); - global_data.used_memory_ranges.grow_capacity(global_data.used_memory_ranges.size() + state.reg.size()); + global_data.physical_memory_ranges.try_grow_capacity(global_data.physical_memory_ranges.size() + state.reg.size()).release_value_but_fixme_should_propagate_errors(); + global_data.used_memory_ranges.try_grow_capacity(global_data.used_memory_ranges.size() + state.reg.size()).release_value_but_fixme_should_propagate_errors(); for (auto const& reg_entry : state.reg) { dbgln("MM: Reserved Range {}: address: {} size {:#x}", node_name, PhysicalAddress { reg_entry.start_addr }, reg_entry.size); @@ -640,7 +640,7 @@ UNMAP_AFTER_INIT void MemoryManager::parse_memory_map_fdt(MemoryManager::GlobalD VERIFY(state.address_cells); VERIFY(state.size_cells); - state.reg.ensure_capacity(data.size() / ((state.address_cells + state.size_cells) * sizeof(u32))); + state.reg.try_ensure_capacity(data.size() / ((state.address_cells + state.size_cells) * sizeof(u32))).release_value_but_fixme_should_propagate_errors(); FixedMemoryStream reg_stream { data }; @@ -695,7 +695,7 @@ UNMAP_AFTER_INIT void MemoryManager::parse_memory_map_multiboot(MemoryManager::G if (g_boot_info.boot_method_specific.multiboot1.flags & 0x4 && !g_boot_info.boot_method_specific.multiboot1.module_physical_ptr.is_null()) { dmesgln("MM: Multiboot module @ {}, length={}", g_boot_info.boot_method_specific.multiboot1.module_physical_ptr, g_boot_info.boot_method_specific.multiboot1.module_length); VERIFY(g_boot_info.boot_method_specific.multiboot1.module_length != 0); - global_data.used_memory_ranges.append(UsedMemoryRange { UsedMemoryRangeType::BootModule, g_boot_info.boot_method_specific.multiboot1.module_physical_ptr, g_boot_info.boot_method_specific.multiboot1.module_physical_ptr.offset(g_boot_info.boot_method_specific.multiboot1.module_length) }); + global_data.used_memory_ranges.try_append(UsedMemoryRange { UsedMemoryRangeType::BootModule, g_boot_info.boot_method_specific.multiboot1.module_physical_ptr, g_boot_info.boot_method_specific.multiboot1.module_physical_ptr.offset(g_boot_info.boot_method_specific.multiboot1.module_length) }).release_value_but_fixme_should_propagate_errors(); } auto const* mmap_begin = g_boot_info.boot_method_specific.multiboot1.memory_map; @@ -712,7 +712,7 @@ UNMAP_AFTER_INIT void MemoryManager::parse_memory_map_multiboot(MemoryManager::G auto start_address = PhysicalAddress(address); switch (mmap->type) { case (MULTIBOOT_MEMORY_AVAILABLE): - global_data.physical_memory_ranges.append(PhysicalMemoryRange { PhysicalMemoryRangeType::Usable, start_address, length }); + global_data.physical_memory_ranges.try_append(PhysicalMemoryRange { PhysicalMemoryRangeType::Usable, start_address, length }).release_value_but_fixme_should_propagate_errors(); break; case (MULTIBOOT_MEMORY_RESERVED): #if ARCH(X86_64) @@ -734,21 +734,21 @@ UNMAP_AFTER_INIT void MemoryManager::parse_memory_map_multiboot(MemoryManager::G if ((address != 0x000000fd00000000 || length != (0x000000ffffffffff - 0x000000fd00000000) + 1) && (address != 0x000003fff0000000 || length != 0x10000000)) #endif - global_data.physical_memory_ranges.append(PhysicalMemoryRange { PhysicalMemoryRangeType::Reserved, start_address, length }); + global_data.physical_memory_ranges.try_append(PhysicalMemoryRange { PhysicalMemoryRangeType::Reserved, start_address, length }).release_value_but_fixme_should_propagate_errors(); break; case (MULTIBOOT_MEMORY_ACPI_RECLAIMABLE): - global_data.physical_memory_ranges.append(PhysicalMemoryRange { PhysicalMemoryRangeType::ACPI_Reclaimable, start_address, length }); + global_data.physical_memory_ranges.try_append(PhysicalMemoryRange { PhysicalMemoryRangeType::ACPI_Reclaimable, start_address, length }).release_value_but_fixme_should_propagate_errors(); break; case (MULTIBOOT_MEMORY_NVS): - global_data.physical_memory_ranges.append(PhysicalMemoryRange { PhysicalMemoryRangeType::ACPI_NVS, start_address, length }); + global_data.physical_memory_ranges.try_append(PhysicalMemoryRange { PhysicalMemoryRangeType::ACPI_NVS, start_address, length }).release_value_but_fixme_should_propagate_errors(); break; case (MULTIBOOT_MEMORY_BADRAM): dmesgln("MM: Warning, detected bad memory range!"); - global_data.physical_memory_ranges.append(PhysicalMemoryRange { PhysicalMemoryRangeType::BadMemory, start_address, length }); + global_data.physical_memory_ranges.try_append(PhysicalMemoryRange { PhysicalMemoryRangeType::BadMemory, start_address, length }).release_value_but_fixme_should_propagate_errors(); break; default: dbgln("MM: Unknown range!"); - global_data.physical_memory_ranges.append(PhysicalMemoryRange { PhysicalMemoryRangeType::Unknown, start_address, length }); + global_data.physical_memory_ranges.try_append(PhysicalMemoryRange { PhysicalMemoryRangeType::Unknown, start_address, length }).release_value_but_fixme_should_propagate_errors(); break; } } @@ -821,7 +821,7 @@ UNMAP_AFTER_INIT void MemoryManager::initialize_physical_pages(GlobalData& globa } else { global_data.physical_pages_region = found_region->try_take_pages_from_beginning(physical_page_array_pages_and_page_tables_count); } - global_data.used_memory_ranges.append({ UsedMemoryRangeType::PhysicalPages, global_data.physical_pages_region->lower(), global_data.physical_pages_region->upper() }); + global_data.used_memory_ranges.try_append({ UsedMemoryRangeType::PhysicalPages, global_data.physical_pages_region->lower(), global_data.physical_pages_region->upper() }).release_value_but_fixme_should_propagate_errors(); // Create the bare page directory. This is not a fully constructed page directory and merely contains the allocators! m_kernel_page_directory = PageDirectory::must_create_kernel_page_directory(); diff --git a/Kernel/Memory/PhysicalRegion.cpp b/Kernel/Memory/PhysicalRegion.cpp index 004bdd2178c18e..3c3f22903f9b7f 100644 --- a/Kernel/Memory/PhysicalRegion.cpp +++ b/Kernel/Memory/PhysicalRegion.cpp @@ -44,7 +44,8 @@ void PhysicalRegion::initialize_zones() size_t zone_count = 0; auto first_address = base_address; while (remaining_pages >= pages_per_zone) { - m_zones.append(adopt_nonnull_own_or_enomem(new (nothrow) PhysicalZone(base_address, pages_per_zone)).release_value_but_fixme_should_propagate_errors()); + auto zone = adopt_nonnull_own_or_enomem(new (nothrow) PhysicalZone(base_address, pages_per_zone)).release_value_but_fixme_should_propagate_errors(); + m_zones.try_append(move(zone)).release_value_but_fixme_should_propagate_errors(); base_address = base_address.offset(pages_per_zone * PAGE_SIZE); m_usable_zones.append(*m_zones.last()); remaining_pages -= pages_per_zone; @@ -95,10 +96,10 @@ Vector> PhysicalRegion::take_contiguous_free_page return {}; Vector> physical_pages; - physical_pages.ensure_capacity(count); + physical_pages.try_ensure_capacity(count).release_value_but_fixme_should_propagate_errors(); for (size_t i = 0; i < count; ++i) - physical_pages.append(PhysicalRAMPage::create(page_base.value().offset(i * PAGE_SIZE))); + physical_pages.unchecked_append(PhysicalRAMPage::create(page_base.value().offset(i * PAGE_SIZE))); return physical_pages; } diff --git a/Kernel/Memory/SharedInodeVMObject.cpp b/Kernel/Memory/SharedInodeVMObject.cpp index 4e2839a5137945..084cff663ccb7a 100644 --- a/Kernel/Memory/SharedInodeVMObject.cpp +++ b/Kernel/Memory/SharedInodeVMObject.cpp @@ -71,7 +71,7 @@ ErrorOr SharedInodeVMObject::sync_impl(off_t offset_in_pages, size_t pages for (size_t page_index = offset_in_pages; page_index < highest_page_to_flush; ++page_index) { auto& physical_page = m_physical_pages[page_index]; if (physical_page && is_page_dirty(page_index)) - pages_to_flush.append(page_index); + TRY(pages_to_flush.try_append(page_index)); } if (pages_to_flush.size() == 0) diff --git a/Kernel/Memory/VirtualRange.cpp b/Kernel/Memory/VirtualRange.cpp index eea79f785a077f..5a50c57ea9b036 100644 --- a/Kernel/Memory/VirtualRange.cpp +++ b/Kernel/Memory/VirtualRange.cpp @@ -19,9 +19,9 @@ Vector VirtualRange::carve(VirtualRange const& taken) const if (taken == *this) return {}; if (taken.base() > base()) - parts.append({ base(), taken.base().get() - base().get() }); + parts.unchecked_append({ base(), taken.base().get() - base().get() }); if (taken.end() < end()) - parts.append({ taken.end(), end().get() - taken.end().get() }); + parts.unchecked_append({ taken.end(), end().get() - taken.end().get() }); return parts; } diff --git a/Kernel/Net/IP/IP.h b/Kernel/Net/IP/IP.h index 233478e480b33d..ddcc85dca8e631 100644 --- a/Kernel/Net/IP/IP.h +++ b/Kernel/Net/IP/IP.h @@ -22,32 +22,4 @@ enum class TransportProtocol : u8 { ICMPv6 = 58, }; -class InternetChecksum { -public: - void add(ReadonlyBytes bytes) - { - VERIFY(!m_uneven_payload); - Span words { bit_cast(bytes.data()), bytes.size() / 2 }; - for (u16 w : words) { - m_checksum += AK::convert_between_host_and_network_endian(w); - if (m_checksum & 0x80000000) - m_checksum = (m_checksum & 0xffff) | (m_checksum >> 16); - } - if (bytes.size() % 2 == 1) { - m_checksum += (bytes.last()) << 8; - m_uneven_payload = true; - } - } - NetworkOrdered finish() - { - while (m_checksum >> 16) - m_checksum = (m_checksum & 0xffff) + (m_checksum >> 16); - return ~m_checksum & 0xffff; - } - -private: - u32 m_checksum { 0 }; - bool m_uneven_payload { false }; -}; - } diff --git a/Kernel/Net/IP/IPv4.h b/Kernel/Net/IP/IPv4.h index 0daee35ccdae4d..9e54b2c75a46aa 100644 --- a/Kernel/Net/IP/IPv4.h +++ b/Kernel/Net/IP/IPv4.h @@ -9,6 +9,7 @@ #include #include #include +#include #include #include @@ -81,9 +82,7 @@ class [[gnu::packed]] IPv4Packet { NetworkOrdered compute_checksum() const { VERIFY(!m_checksum); - InternetChecksum checksum; - checksum.add({ this, sizeof(IPv4Packet) }); - return checksum.finish(); + return InternetChecksum({ this, sizeof(IPv4Packet) }).digest(); } private: diff --git a/Kernel/Net/IP/Socket.cpp b/Kernel/Net/IP/Socket.cpp index 682f8f0432c158..ecbe74262e0ed9 100644 --- a/Kernel/Net/IP/Socket.cpp +++ b/Kernel/Net/IP/Socket.cpp @@ -556,7 +556,7 @@ ErrorOr IPv4Socket::setsockopt(int level, int option, Userspace>; @@ -331,10 +336,10 @@ void handle_icmpv6(EthernetFrameHeader const& eth, IPv6PacketHeader const& ipv6_ header.next_header = TransportProtocol::ICMPv6; InternetChecksum checksum; - checksum.add({ &header, sizeof(IPv6PseudoHeader) }); - checksum.add({ &response, sizeof(advertisement_with_option) }); + checksum.update({ &header, sizeof(IPv6PseudoHeader) }); + checksum.update({ &response, sizeof(advertisement_with_option) }); - response.base.header.set_checksum(checksum.finish()); + response.base.header.set_checksum(checksum.digest()); } else if (icmpv6_header.type() == ICMPv6Type::EchoRequest) { dbgln_if(ICMPV6_DEBUG, "handle_icmp6: got echo request"); if (icmp_packet_size < sizeof(ICMPv6Echo)) { @@ -370,10 +375,10 @@ void handle_icmpv6(EthernetFrameHeader const& eth, IPv6PacketHeader const& ipv6_ memcpy(response.payload(), request.payload(), icmp_packet_size - sizeof(ICMPv6Echo)); InternetChecksum checksum; - checksum.add({ &header, sizeof(IPv6PseudoHeader) }); - checksum.add({ &response, icmp_packet_size }); + checksum.update({ &header, sizeof(IPv6PseudoHeader) }); + checksum.update({ &response, icmp_packet_size }); - response.header.set_checksum(checksum.finish()); + response.header.set_checksum(checksum.digest()); } else { dbgln_if(ICMPV6_DEBUG, "handle_icmp6: got unknown ICMPv6 type {:#02x}", icmpv6_header.type()); return; @@ -396,7 +401,7 @@ void handle_icmp(EthernetFrameHeader const& eth, IPv4Packet const& ipv4_packet, IPv4Socket::all_sockets().with_exclusive([&](auto& sockets) { for (auto& socket : sockets) { if (socket.protocol() == (unsigned)TransportProtocol::ICMP) - icmp_sockets.append(socket); + icmp_sockets.try_append(socket).release_value_but_fixme_should_propagate_errors(); } }); for (auto& socket : icmp_sockets) @@ -431,9 +436,7 @@ void handle_icmp(EthernetFrameHeader const& eth, IPv4Packet const& ipv4_packet, if (icmp_packet_size > icmp_header_size) memcpy(response.payload, request.payload, icmp_packet_size - icmp_header_size); - InternetChecksum checksum; - checksum.add({ &response, icmp_packet_size }); - response.header.checksum = checksum.finish(); + response.header.checksum = InternetChecksum({ &response, icmp_packet_size }).digest(); adapter->send_packet(packet->bytes()); adapter->release_packet_buffer(*packet); diff --git a/Kernel/Net/NetworkTask.h b/Kernel/Net/NetworkTask.h index 821c85b4308f21..b9877836ce6885 100644 --- a/Kernel/Net/NetworkTask.h +++ b/Kernel/Net/NetworkTask.h @@ -6,10 +6,13 @@ #pragma once +#include namespace Kernel { + class NetworkTask { public: static void spawn(); static bool is_current(); + static Thread* the_thread(); }; } diff --git a/Kernel/Net/NetworkingManagement.cpp b/Kernel/Net/NetworkingManagement.cpp index c6f624964da2e0..730fa30ddbb1ec 100644 --- a/Kernel/Net/NetworkingManagement.cpp +++ b/Kernel/Net/NetworkingManagement.cpp @@ -13,6 +13,7 @@ #include #include #include +#include #include #include #include @@ -136,6 +137,28 @@ UNMAP_AFTER_INIT ErrorOr> NetworkingManagement::de return Error::from_errno(ENODEV); } +ErrorOr NetworkingManagement::register_adapter(NonnullRefPtr adapter) +{ + TRY(m_adapters.with([&](auto& adapters) -> ErrorOr { + return adapters.try_append(adapter); + })); + TRY(adapter->initialize({})); + + // HACK: If we the Network Task is already running, + // we need to do set the on_receive callback to the one used by the Network Task. + if (auto* network_task_thread = NetworkTask::the_thread(); network_task_thread && network_task_thread->state() != Thread::State::Dying) { + dmesgln("NetworkingManagement: Registering network adapter {} while Network Task is running, hooking up on_receive callback", adapter->name()); + // Note: We always have another network adapter registered before this one, + // so we can grab the on_receive callback from adapter 0 + auto& on_wake = m_adapters.with([&](auto& adapters) -> auto& { + return adapters.first()->on_receive; + }); + adapter->on_receive = [&on_wake] { on_wake(); }; + } + + return {}; +} + bool NetworkingManagement::initialize() { if (!kernel_command_line().is_physical_networking_disabled() && !PCI::Access::is_disabled()) { @@ -148,11 +171,11 @@ bool NetworkingManagement::initialize() dmesgln("Failed to initialize network adapter ({} {}): {}", device_identifier.address(), device_identifier.hardware_id(), result.error()); return; } - m_adapters.with([&](auto& adapters) { adapters.append(*result.release_value()); }); + m_adapters.with([&](auto& adapters) { adapters.try_append(*result.release_value()).release_value_but_fixme_should_propagate_errors(); }); })); } auto loopback = MUST(LoopbackAdapter::try_create()); - m_adapters.with([&](auto& adapters) { adapters.append(*loopback); }); + m_adapters.with([&](auto& adapters) { adapters.try_append(*loopback).release_value_but_fixme_should_propagate_errors(); }); m_loopback_adapter = *loopback; return true; } diff --git a/Kernel/Net/NetworkingManagement.h b/Kernel/Net/NetworkingManagement.h index 74574c315100dc..24be6a6ffd3604 100644 --- a/Kernel/Net/NetworkingManagement.h +++ b/Kernel/Net/NetworkingManagement.h @@ -39,6 +39,8 @@ class NetworkingManagement { NonnullRefPtr loopback_adapter() const; + ErrorOr register_adapter(NonnullRefPtr); + private: ErrorOr> determine_network_device(PCI::DeviceIdentifier const&) const; diff --git a/Kernel/Net/Realtek/RTL8168NetworkAdapter.cpp b/Kernel/Net/Realtek/RTL8168NetworkAdapter.cpp index 0bded1875a17c5..313313fb1184aa 100644 --- a/Kernel/Net/Realtek/RTL8168NetworkAdapter.cpp +++ b/Kernel/Net/Realtek/RTL8168NetworkAdapter.cpp @@ -1257,7 +1257,7 @@ UNMAP_AFTER_INIT void RTL8168NetworkAdapter::initialize_rx_descriptors() auto& descriptor = m_rx_descriptors[i]; auto region = MM.allocate_contiguous_kernel_region(Memory::page_round_up(RX_BUFFER_SIZE).release_value_but_fixme_should_propagate_errors(), "RTL8168 RX buffer"sv, Memory::Region::Access::ReadWrite).release_value(); memset(region->vaddr().as_ptr(), 0, region->size()); // MM already zeros out newly allocated pages, but we do it again in case that ever changes - m_rx_buffers_regions.append(move(region)); + m_rx_buffers_regions.try_append(move(region)).release_value_but_fixme_should_propagate_errors(); descriptor.buffer_size = RX_BUFFER_SIZE; descriptor.flags = RXDescriptor::Ownership; // let the NIC know it can use this descriptor @@ -1274,7 +1274,7 @@ UNMAP_AFTER_INIT void RTL8168NetworkAdapter::initialize_tx_descriptors() auto& descriptor = m_tx_descriptors[i]; auto region = MM.allocate_contiguous_kernel_region(Memory::page_round_up(TX_BUFFER_SIZE).release_value_but_fixme_should_propagate_errors(), "RTL8168 TX buffer"sv, Memory::Region::Access::ReadWrite).release_value(); memset(region->vaddr().as_ptr(), 0, region->size()); // MM already zeros out newly allocated pages, but we do it again in case that ever changes - m_tx_buffers_regions.append(move(region)); + m_tx_buffers_regions.try_append(move(region)).release_value_but_fixme_should_propagate_errors(); descriptor.flags = TXDescriptor::FirstSegment | TXDescriptor::LastSegment; auto physical_address = m_tx_buffers_regions[i]->physical_page(0)->paddr().get(); diff --git a/Kernel/Net/USB/CDCECM.cpp b/Kernel/Net/USB/CDCECM.cpp new file mode 100644 index 00000000000000..44babb626691f8 --- /dev/null +++ b/Kernel/Net/USB/CDCECM.cpp @@ -0,0 +1,245 @@ +/* + * Copyright (c) 2025, Leon Albrecht + * + * SPDX-License-Identifier: BSD-2-Clause + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace Kernel::USB::CDC { + +ErrorOr create_ecm_network_adapter(USB::Device& device, USB::USBInterface const& control, Vector const& data_interface_ids) +{ + Optional inactive_data_interface; + Optional active_data_interface; + if (data_interface_ids.size() == 0) { + dmesgln("CDC-ECM: Interface {} has no associated data interfaces; Rejecting", control.descriptor().interface_id); + return ENOTSUP; + } + for (auto interface_id : data_interface_ids) { + // FIXME: Maybe make this a direct query? + for (auto const& interface : control.configuration().interfaces()) { + if (interface.descriptor().interface_id != interface_id) + continue; + + // The active interface is the one that has two endpoints + // While the inactive one has no endpoints + auto endpoint_count = interface.descriptor().number_of_endpoints; + if (endpoint_count == 2) { + active_data_interface = interface; + } else if (endpoint_count == 0) { + inactive_data_interface = interface; + } else { + dmesgln("CDC-ECM: Data interface {}.{} has invalid number of endpoints: {}; Rejecting", interface_id, interface.descriptor().alternate_setting, endpoint_count); + return ENOTSUP; + } + } + } + + if (!inactive_data_interface.has_value() || !active_data_interface.has_value()) { + dmesgln("CDC-ECM: Could not find both active and inactive data interfaces; Rejecting"); + return ENOTSUP; + } + auto adapter = TRY(CDCECMNetworkAdapter::create( + device, + control, + inactive_data_interface.value(), + active_data_interface.value())); + TRY(NetworkingManagement::the().register_adapter(adapter)); + dmesgln("CDC-ECM: Successfully initialized CDC-ECM network adapter"); + return {}; +} + +ErrorOr> CDCECMNetworkAdapter::create(USB::Device& device, USB::USBInterface const& control, USB::USBInterface const& data_inactive, USB::USBInterface const& data_active) +{ + u16 max_segment_size; + u8 mac_string_index; + TRY(control.configuration().for_each_descriptor_in_interface(control, [&](ReadonlyBytes raw_descriptor) -> ErrorOr { + auto const& descriptor_header = *reinterpret_cast(raw_descriptor.data()); + if (descriptor_header.descriptor_type != static_cast(CDC::ClassSpecificDescriptorCodes::CS_Interface)) + return IterationDecision::Continue; + + auto subtype = raw_descriptor[2]; + if (static_cast(subtype) != CDC::ClassSpecificInterfaceDescriptorCodes::EthernetNetworking) + return IterationDecision::Continue; + + auto stream = FixedMemoryStream(raw_descriptor.slice(sizeof(USBDescriptorCommon) + 1)); + mac_string_index = TRY(stream.read_value()); + [[maybe_unused]] u32 bm_stats = TRY(stream.read_value>()); + max_segment_size = TRY(stream.read_value>()); + [[maybe_unused]] u16 w_number_mc_filters = TRY(stream.read_value>()); + [[maybe_unused]] u8 b_number_power_filters = TRY(stream.read_value()); + return IterationDecision::Break; + })); + + auto mac_string = TRY(device.get_string_descriptor(mac_string_index)); + if (mac_string->length() != 12) { + dbgln("USB CDC-ECM: Invalid MAC address string length: {}", mac_string->length()); + return EINVAL; + } + MACAddress mac_address; + for (size_t i = 0; i < 6; ++i) { + auto byte_str = mac_string->view().substring_view(i * 2, 2); + auto byte_value = AK::StringUtils::convert_to_uint_from_hex(byte_str); + if (!byte_value.has_value()) { + dbgln("USB CDC-ECM: Invalid MAC address string: {}", mac_string->view()); + return EINVAL; + } + mac_address[i] = byte_value.value(); + } + dmesgln("USB CDC-ECM: Using MAC address: {}", TRY(mac_address.to_string())); + + TRY(device.set_configuration(control.configuration())); + + u8 in_pipe_endpoint_number = 0xff; + u16 in_max_packet_size; + u8 out_pipe_endpoint_number = 0xff; + u16 out_max_packet_size; + + if (data_active.descriptor().number_of_endpoints < 2) { + dmesgln("CDC-ECM: Data Interface does not provide enough endpoints; Rejecting"); + return ENOTSUP; + } + + for (auto const& endpoint : data_active.endpoints()) { + if (endpoint.endpoint_attributes_bitmap != USBEndpoint::ENDPOINT_ATTRIBUTES_TRANSFER_TYPE_BULK) + continue; + // The upper bit of the Endpoint address is set to 1, iff it is the Bulk-In Endpoint + if (endpoint.endpoint_address & 0x80) { + in_pipe_endpoint_number = endpoint.endpoint_address & 0b1111; + in_max_packet_size = endpoint.max_packet_size; + } else { + out_pipe_endpoint_number = endpoint.endpoint_address & 0b1111; + out_max_packet_size = endpoint.max_packet_size; + } + } + + if (in_pipe_endpoint_number == 0xff || out_pipe_endpoint_number == 0xff) { + // FIXME: We may also get isochronous endpoints, handle those too + dmesgln("CDC-ECM: Data Interface did not advertise two Bulk Endpoints; Rejecting"); + return ENOTSUP; + } + + auto event_endpoint = control.endpoints().first(); + if (event_endpoint.endpoint_attributes_bitmap != USBEndpoint::ENDPOINT_ATTRIBUTES_TRANSFER_TYPE_INTERRUPT) { + dmesgln("CDC-ECM: Control Interface's first endpoint is not an interrupt endpoint; Rejecting"); + return ENOTSUP; + } + + auto event_pipe = TRY(InterruptInPipe::create(device.controller(), device, event_endpoint.endpoint_address & 0b1111, event_endpoint.max_packet_size, event_endpoint.poll_interval_in_frames)); + auto in_pipe = TRY(BulkInPipe::create(device.controller(), device, in_pipe_endpoint_number, in_max_packet_size)); + auto out_pipe = TRY(BulkOutPipe::create(device.controller(), device, out_pipe_endpoint_number, out_max_packet_size)); + + // FIXME: Maybe also set-up the notification interrupt pipe from the control interface + + return adopt_nonnull_ref_or_enomem(new (nothrow) CDCECMNetworkAdapter( + device, + mac_address, + move(event_pipe), + move(in_pipe), + move(out_pipe), + data_active, + data_inactive, + max_segment_size)); +} + +CDCECMNetworkAdapter::CDCECMNetworkAdapter(USB::Device& device, + MACAddress const& mac_address, + NonnullOwnPtr event_pipe, + NonnullOwnPtr in_pipe, + NonnullOwnPtr out_pipe, + USBInterface const& active_data_interface, + USBInterface const& inactive_data_interface, + u16 max_segment_size) + : NetworkAdapter("cdc-ecm"sv) // FIXME: Choose the propper name + // FIXME: We may want to make this unique if we have multiple CDC-ECM devices + , m_device(device) + , m_event_pipe(move(event_pipe)) + , m_in_pipe(move(in_pipe)) + , m_out_pipe(move(out_pipe)) + , m_active_data_interface(active_data_interface) + , m_inactive_data_interface(inactive_data_interface) +{ + set_mtu(max_segment_size); + set_mac_address(mac_address); +} + +CDCECMNetworkAdapter::~CDCECMNetworkAdapter() = default; + +ErrorOr CDCECMNetworkAdapter::initialize(Badge) +{ + dmesgln("CDC-ECM: Activating data interface {}.{}", m_active_data_interface.descriptor().interface_id, m_active_data_interface.descriptor().alternate_setting); + TRY(m_device.set_configuration_and_interface(m_active_data_interface)); + + // FIXME: Also listen to the event pipe for notifications + // Note: This would need to be done before activating the alternative interface + auto [process, poll_thread] = TRY(Process::create_kernel_process("CDC-ECM"sv, [this]() { this->poll_thread(); })); + poll_thread->set_name("CDC-ECM Poll"sv); + m_process = move(process); + + return {}; +} + +bool CDCECMNetworkAdapter::link_up() +{ + // FIXME: Listen for a NetworkConnection Notification? + // Note: We would likely need to listen to that before activating the alternative interface + // As that seems to be the step that initializes the ethernet controller + return true; +} + +i32 CDCECMNetworkAdapter::link_speed() +{ + // FIXME: Listen for a ConnectionSpeedChange Notification + // Note: See above + return 1000; // Let's assume gigabit for now +} + +void CDCECMNetworkAdapter::send_raw(ReadonlyBytes packet) +{ + // FIXME: Handle errors properly + // Note: CDC-ECM requires that a frame is ended by a short packet, + // Splitting of the packet is handled by the controller or host-driver. + // So we only need to optionally insert a zero-length packet at the end + // in case the sent packet is an exact multiple of the max transfer size. + size_t const max_transfer_size = m_out_pipe->max_packet_size(); + m_out_pipe->submit_bulk_out_transfer(packet.size(), packet.data()).release_value_but_fixme_should_propagate_errors(); + if (packet.size() % max_transfer_size == 0) { + // Send a zero-length packet to indicate end of frame + m_out_pipe->submit_bulk_out_transfer(0, nullptr).release_value_but_fixme_should_propagate_errors(); + } +} + +void CDCECMNetworkAdapter::poll_thread() +{ + // FIXME: Listen for a ResponseAvailable Notification + size_t const max_packet_size = m_in_pipe->max_packet_size(); + auto buffer = ByteBuffer::create_uninitialized(mtu()).release_value_but_fixme_should_propagate_errors(); + // Note: The stitching is most likely not needed, as the USB controller should + // already stitch packets together for us until it hits a short packet, + // or the buffer is full. (which's size matches the device advertised MTU/max segment size, so it should be fine) + size_t offset = 0; + while (!Process::current().is_dying()) { + size_t received_length = m_in_pipe->submit_bulk_in_transfer(buffer.size() - offset, buffer.data() + offset).release_value_but_fixme_should_propagate_errors(); + if (received_length != max_packet_size) { + // A small packet indicates the end of a frame (this also handles pre-stitched frames) + did_receive(buffer.span().slice(0, offset + received_length)); + offset = 0; + } else { + // FIXME: When the received frame is exactly max packet size and the controller did the stitching, + // we would wait for a short packet that never arrives, or append data from the next frame. + dmesgln("CDC-ECM: Stitching frame"); + offset += received_length; + } + } +} + +} diff --git a/Kernel/Net/USB/CDCECM.h b/Kernel/Net/USB/CDCECM.h new file mode 100644 index 00000000000000..5a098a53ca41ed --- /dev/null +++ b/Kernel/Net/USB/CDCECM.h @@ -0,0 +1,62 @@ +/* + * Copyright (c) 2025, Leon Albrecht + * + * SPDX-License-Identifier: BSD-2-Clause + */ + +#pragma once + +#include +#include +#include +#include +#include + +namespace Kernel::USB::CDC { + +class CDCECMNetworkAdapter final : public NetworkAdapter { +public: + static ErrorOr> create(USB::Device&, USB::USBInterface const& control, USB::USBInterface const& data_inactive, USB::USBInterface const& data_active); + + virtual ~CDCECMNetworkAdapter() override; + + virtual StringView class_name() const override { return "CDCECMNetworkAdapter"sv; } + virtual Type adapter_type() const override { return Type::Ethernet; } + virtual ErrorOr initialize(Badge) override; + + virtual bool link_up() override; + virtual i32 link_speed() override; + virtual bool link_full_duplex() override + { + return true; // FIXME: Is this always true? + } + +protected: + virtual void send_raw(ReadonlyBytes) override; + +private: + CDCECMNetworkAdapter(USB::Device&, + MACAddress const& mac_address, + NonnullOwnPtr event_pipe, + NonnullOwnPtr in_pipe, + NonnullOwnPtr out_pipe, + USBInterface const& active_data_interface, + USBInterface const& inactive_data_interface, + u16 max_segment_size); + + USB::Device& m_device; + + NonnullOwnPtr m_event_pipe; + NonnullOwnPtr m_in_pipe; + NonnullOwnPtr m_out_pipe; + USBInterface const& m_active_data_interface; + USBInterface const& m_inactive_data_interface; + + RefPtr m_process; + + void poll_thread(); +}; + +ErrorOr create_ecm_network_adapter(USB::Device&, USB::USBInterface const& control, Vector const& data_interface_ids); + +} diff --git a/Kernel/Syscalls/SyscallHandler.cpp b/Kernel/Syscalls/SyscallHandler.cpp index 59279cfec7679b..52e8cded56fda6 100644 --- a/Kernel/Syscalls/SyscallHandler.cpp +++ b/Kernel/Syscalls/SyscallHandler.cpp @@ -170,7 +170,12 @@ NEVER_INLINE void syscall_handler(TrapFrame* trap) current_thread->yield_if_should_be_stopped(); - current_thread->check_dispatch_pending_signal(); +#if ARCH(X86_64) + // On x86-64, we don't go through ProcessorBase::exit_trap when returning to userspace + // (except in the SC_sigreturn case), so we dispatch signals here instead. + if (function != SC_sigreturn) + current_thread->check_dispatch_pending_signal(); +#endif // If the previous mode somehow changed something is seriously messed up... VERIFY(current_thread->previous_mode() == ExecutionMode::User); diff --git a/Kernel/Tasks/Process.cpp b/Kernel/Tasks/Process.cpp index 84119dea9edbe6..b1aed80ab87f15 100644 --- a/Kernel/Tasks/Process.cpp +++ b/Kernel/Tasks/Process.cpp @@ -831,10 +831,19 @@ ErrorOr Process::dump_core() return {}; } auto coredump_path = TRY(name().with([&](auto& process_name) { - return KString::formatted("{}/{}_{}_{}", coredump_directory_path->view(), process_name.representable_view(), pid().value(), kgettimeofday().seconds_since_epoch()); + return KString::formatted("{}/{}_{}_{}.partial", coredump_directory_path->view(), process_name.representable_view(), pid().value(), kgettimeofday().seconds_since_epoch()); })); auto coredump = TRY(Coredump::try_create(*this, coredump_path->view())); - return coredump->write(); + TRY(coredump->write()); + + auto root_custody = vfs_root_context()->root_custody().with([](auto& custody) -> NonnullRefPtr { + return custody; + }); + + auto new_path = TRY(KString::try_create(coredump_path->view().trim(".partial"sv))); + TRY(VirtualFileSystem::rename(vfs_root_context(), credentials(), *root_custody, coredump_path->view(), *root_custody, new_path->view())); + + return {}; } ErrorOr Process::dump_perfcore() diff --git a/Kernel/Tasks/Scheduler.cpp b/Kernel/Tasks/Scheduler.cpp index bea1e13c8ef5ff..79498586752b78 100644 --- a/Kernel/Tasks/Scheduler.cpp +++ b/Kernel/Tasks/Scheduler.cpp @@ -201,7 +201,7 @@ UNMAP_AFTER_INIT void Scheduler::start() VERIFY_NOT_REACHED(); } -ShouldYield Scheduler::pick_next() +ScheduleResult Scheduler::pick_next() { VERIFY_INTERRUPTS_DISABLED(); @@ -236,11 +236,20 @@ ShouldYield Scheduler::pick_next() // but since we're still holding the scheduler lock we're still in a critical section critical.leave(); + auto* previous_thread = Thread::current(); + thread_to_schedule.set_ticks_left(time_slice_for(thread_to_schedule)); - return context_switch(&thread_to_schedule); + context_switch(&thread_to_schedule); + + if (previous_thread == &thread_to_schedule) { + VERIFY(thread_to_schedule.is_idle_thread()); + return ScheduleResult::NoRunnableThreadFound; + } + + return ScheduleResult::Success; } -void Scheduler::yield() +ScheduleResult Scheduler::yield() { InterruptDisabler disabler; @@ -252,15 +261,13 @@ void Scheduler::yield() // a critical section where we don't want to switch contexts, then // delay until exiting the trap or critical section Processor::current().invoke_scheduler_async(); - return; + return ScheduleResult::Delayed; } - auto result = pick_next(); - while (result == ShouldYield::Yes) - result = pick_next(); + return pick_next(); } -ShouldYield Scheduler::context_switch(Thread* thread) +void Scheduler::context_switch(Thread* thread) { thread->did_schedule(); @@ -268,7 +275,7 @@ ShouldYield Scheduler::context_switch(Thread* thread) VERIFY(from_thread); if (from_thread == thread) - return ShouldYield::No; + return; // If the last process hasn't blocked (still marked as running), // mark it as runnable for the next round, unless it's supposed @@ -303,11 +310,6 @@ ShouldYield Scheduler::context_switch(Thread* thread) // switched from, and thread reflects Thread::current() enter_current(*from_thread); VERIFY(thread == Thread::current()); - - { - SpinlockLocker lock(thread->get_lock()); - return thread->dispatch_one_pending_signal() == DispatchSignalResult::Yield ? ShouldYield::Yes : ShouldYield::No; - } } void Scheduler::enter_current(Thread& prev_thread) @@ -471,11 +473,8 @@ void Scheduler::invoke_async() // Since this function is called when leaving critical sections (such // as a Spinlock), we need to check if we're not already doing this // to prevent recursion - if (!Processor::current_in_scheduler()) { - auto result = pick_next(); - while (result == ShouldYield::Yes) - result = pick_next(); - } + if (!Processor::current_in_scheduler()) + pick_next(); } void Scheduler::notify_finalizer() @@ -488,14 +487,27 @@ void Scheduler::idle_loop(void*) { auto& proc = Processor::current(); dbgln("Scheduler[{}]: idle loop running", proc.id()); - VERIFY(Processor::are_interrupts_enabled()); + + // Interrupts have to be disabled during the idle loop to prevent lost wakeups. + // If interrupts were enabled between yield() and proc.idle(), we could get an interrupt between those two function calls, + // but still go to sleep, even if the interrupt caused a thread to be runnable or notified us of one. + InterruptDisabler disabler; for (;;) { - proc.idle_begin(); - proc.wait_for_interrupt(); - proc.idle_end(); - VERIFY_INTERRUPTS_ENABLED(); - yield(); + // First, check if there is a runnable thread we can switch to. + auto result = yield(); + + if (result == ScheduleResult::NoRunnableThreadFound) { + // If there is no runnable thread, go to sleep until we get an interrupt. + + // This function causes the processor to go to sleep while interrupts are still disabled. + // After going to sleep, it will listen for interrupts, and if it receives one, it will wake up again. + // Subsequently, the interrupt handler for the received interrupt will be called. + proc.idle(); + + // This interrupt might have caused a new thread to become runnable or notified us of one. + // So check for runnable threads in the next loop iteration again. + } } } diff --git a/Kernel/Tasks/Scheduler.h b/Kernel/Tasks/Scheduler.h index 81feca955eb723..9df26732fbb568 100644 --- a/Kernel/Tasks/Scheduler.h +++ b/Kernel/Tasks/Scheduler.h @@ -30,9 +30,10 @@ struct TotalTimeScheduled { u64 total_kernel { 0 }; }; -enum class [[nodiscard]] ShouldYield { - Yes, - No, +enum class ScheduleResult { + Success, + NoRunnableThreadFound, + Delayed, }; class Scheduler { @@ -42,9 +43,9 @@ class Scheduler { static void set_idle_thread(Thread* idle_thread); static void timer_tick(); [[noreturn]] static void start(); - static ShouldYield pick_next(); - static void yield(); - static ShouldYield context_switch(Thread*); + static ScheduleResult pick_next(); + static ScheduleResult yield(); + static void context_switch(Thread*); static void enter_current(Thread& prev_thread); static void leave_on_first_switch(InterruptsState); static void prepare_after_exec(); diff --git a/Kernel/Tasks/Thread.cpp b/Kernel/Tasks/Thread.cpp index 4ff9a618d5ce27..c35b4525849b6c 100644 --- a/Kernel/Tasks/Thread.cpp +++ b/Kernel/Tasks/Thread.cpp @@ -304,7 +304,7 @@ void Thread::unblock_from_blocker(Blocker& blocker) VERIFY(!is_stopped()); unblock(); }; - if (Processor::current_in_irq() != 0) { + if (Processor::current_in_irq()) { Processor::deferred_call_queue([do_unblock = move(do_unblock), self = try_make_weak_ptr().release_value_but_fixme_should_propagate_errors()]() { if (auto this_thread = self.strong_ref()) do_unblock(); @@ -400,7 +400,7 @@ void Thread::die_if_needed() set_state(Thread::State::Dying); } - VERIFY(Processor::current_in_irq() == 0); + VERIFY(!Processor::current_in_irq()); VERIFY(Processor::in_critical() == 0); Scheduler::yield(); @@ -623,7 +623,7 @@ bool Thread::tick() return m_ticks_left != 0; } -void Thread::check_dispatch_pending_signal() +void Thread::check_dispatch_pending_signal(YieldBehavior yield_behavior) { auto result = DispatchSignalResult::Continue; { @@ -634,7 +634,15 @@ void Thread::check_dispatch_pending_signal() } if (result == DispatchSignalResult::Yield) { - yield_without_releasing_big_lock(); + switch (yield_behavior) { + case YieldBehavior::DoYield: + yield_without_releasing_big_lock(); + break; + case YieldBehavior::FlagYield: + VERIFY_INTERRUPTS_DISABLED(); + Processor::current().invoke_scheduler_async(); + break; + } } } @@ -884,13 +892,9 @@ DispatchSignalResult Thread::dispatch_signal(u8 signal) dbgln_if(SIGNAL_DEBUG, "Dispatch signal {} to {}, state: {}", signal, *this, state_string()); - if (m_state == Thread::State::Invalid || !is_initialized()) { - // Thread has barely been created, we need to wait until it is - // at least in Runnable state and is_initialized() returns true, - // which indicates that it is fully set up an we actually have - // a register state on the stack that we can modify - return DispatchSignalResult::Deferred; - } + // We should only ever dispatch signals to valid and initialized threads. + VERIFY(m_state != Thread::State::Invalid); + VERIFY(is_initialized()); auto& action = m_process->m_signal_action_data[signal]; auto sender_pid = m_signal_senders[signal]; diff --git a/Kernel/Tasks/Thread.h b/Kernel/Tasks/Thread.h index 0dc00bf992ea1f..fd2ad736922e74 100644 --- a/Kernel/Tasks/Thread.h +++ b/Kernel/Tasks/Thread.h @@ -48,6 +48,11 @@ enum class [[nodiscard]] DispatchSignalResult { Continue }; +enum class YieldBehavior { + DoYield, + FlagYield +}; + struct ThreadSpecificData { ThreadSpecificData* self; }; @@ -326,7 +331,7 @@ class Thread SpinlockLocker lock(m_lock); if (!should_add_blocker(blocker, data)) return false; - m_blockers.append({ &blocker, data }); + m_blockers.try_append({ &blocker, data }).release_value_but_fixme_should_propagate_errors(); return true; } @@ -394,9 +399,9 @@ class Thread VERIFY(move_count > 0); Vector taken_blockers; - taken_blockers.ensure_capacity(move_count); + taken_blockers.try_ensure_capacity(move_count).release_value_but_fixme_should_propagate_errors(); for (size_t i = 0; i < move_count; i++) - taken_blockers.append(m_blockers.take(i)); + taken_blockers.unchecked_append(m_blockers.take(i)); m_blockers.remove(0, move_count); return taken_blockers; } @@ -409,9 +414,9 @@ class Thread m_blockers = move(blockers_to_append); return; } - m_blockers.ensure_capacity(m_blockers.size() + blockers_to_append.size()); + m_blockers.try_ensure_capacity(m_blockers.size() + blockers_to_append.size()).release_value_but_fixme_should_propagate_errors(); for (size_t i = 0; i < blockers_to_append.size(); i++) - m_blockers.append(blockers_to_append.take(i)); + m_blockers.unchecked_append(blockers_to_append.take(i)); blockers_to_append.clear(); } @@ -878,7 +883,7 @@ class Thread DispatchSignalResult dispatch_one_pending_signal(); DispatchSignalResult dispatch_signal(u8 signal); - void check_dispatch_pending_signal(); + void check_dispatch_pending_signal(YieldBehavior yield_behavior = YieldBehavior::DoYield); [[nodiscard]] bool has_unmasked_pending_signals() const { return m_have_any_unmasked_pending_signals.load(AK::memory_order_consume); } [[nodiscard]] bool should_ignore_signal(u8 signal) const; [[nodiscard]] bool has_signal_handler(u8 signal) const; @@ -1016,7 +1021,7 @@ class Thread } } if (!have_existing) - m_holding_locks_list.append({ &lock, location, 1 }); + m_holding_locks_list.try_append({ &lock, location, 1 }).release_value_but_fixme_should_propagate_errors(); } else { VERIFY(refs_delta < 0); bool found = false; diff --git a/Kernel/Tasks/ThreadBlockers.cpp b/Kernel/Tasks/ThreadBlockers.cpp index a806a3564f9410..080649cdfe1d10 100644 --- a/Kernel/Tasks/ThreadBlockers.cpp +++ b/Kernel/Tasks/ThreadBlockers.cpp @@ -615,7 +615,7 @@ bool Thread::WaitBlockerSet::unblock(Process& process, WaitBlocker::UnblockFlags } if (!updated_existing) { dbgln_if(WAITBLOCK_DEBUG, "WaitBlockerSet[{}] add {} flags: {}", m_process, process, (int)flags); - m_processes.append(ProcessBlockInfo(process, flags, signal)); + m_processes.try_append(ProcessBlockInfo(process, flags, signal)).release_value_but_fixme_should_propagate_errors(); } } return did_unblock_any; diff --git a/Kernel/Tasks/WaitQueue.cpp b/Kernel/Tasks/WaitQueue.cpp index ea7b1f0ee2e26f..594cbbd2690602 100644 --- a/Kernel/Tasks/WaitQueue.cpp +++ b/Kernel/Tasks/WaitQueue.cpp @@ -48,7 +48,7 @@ void WaitQueue::Waiter::notify(Badge) if (&thread == Thread::current()) { // This can happen if an interrupt handler wakes up the currently running thread. - VERIFY(Processor::current_in_irq() > 0); + VERIFY(Processor::current_in_irq()); thread.set_state(Thread::State::Running); } else { thread.set_state(Thread::State::Runnable); @@ -83,7 +83,7 @@ void WaitQueue::Waiter::maybe_block() if (Processor::in_critical() > 0) PANIC("Attempted to block in critical section"); - if (Processor::current_in_irq() > 0) + if (Processor::current_in_irq()) PANIC("Attempted to block while handling an interrupt"); VERIFY(m_association.has_value()); diff --git a/Kernel/Time/TimeManagement.cpp b/Kernel/Time/TimeManagement.cpp index 3e8ad8411b0ef6..f8b3214886fe8e 100644 --- a/Kernel/Time/TimeManagement.cpp +++ b/Kernel/Time/TimeManagement.cpp @@ -41,7 +41,7 @@ namespace Kernel { -static NeverDestroyed>>> s_recipes; +static NeverDestroyed>> s_hardware_timers; static Singleton s_the; bool TimeManagement::is_initialized() @@ -54,13 +54,13 @@ TimeManagement& TimeManagement::the() return *s_the; } -void TimeManagement::add_recipe(DeviceTree::DeviceRecipe> recipe) +ErrorOr TimeManagement::register_hardware_timer(NonnullLockRefPtr timer) { // This function has to be called before TimeManagement is initialized, // as we do not support dynamic registration of timers. VERIFY(!is_initialized()); - s_recipes->append(move(recipe)); + return s_hardware_timers->try_append(move(timer)); } // The s_scheduler_specific_current_time function provides a current time for scheduling purposes, @@ -328,9 +328,9 @@ UNMAP_AFTER_INIT Vector TimeManagement::scan_and_initialize_ bool should_enable = is_hpet_periodic_mode_allowed(); dbgln("Duration: Scanning for periodic timers"); Vector timers; - for (auto& hardware_timer : m_hardware_timers) { + for (auto& hardware_timer : *s_hardware_timers) { if (hardware_timer->is_periodic_capable()) { - timers.append(hardware_timer); + timers.try_append(hardware_timer).release_value_but_fixme_should_propagate_errors(); if (should_enable) hardware_timer->set_periodic(); } @@ -342,9 +342,9 @@ UNMAP_AFTER_INIT Vector TimeManagement::scan_for_non_periodi { dbgln("Duration: Scanning for non-periodic timers"); Vector timers; - for (auto& hardware_timer : m_hardware_timers) { + for (auto& hardware_timer : *s_hardware_timers) { if (!hardware_timer->is_periodic_capable()) - timers.append(hardware_timer); + timers.try_append(hardware_timer).release_value_but_fixme_should_propagate_errors(); } return timers; } @@ -375,7 +375,7 @@ UNMAP_AFTER_INIT bool TimeManagement::probe_and_set_x86_non_legacy_hardware_time dbgln("HPET: Setting appropriate functions to timers."); for (auto& hpet_comparator : HPET::the().comparators()) - m_hardware_timers.append(hpet_comparator); + s_hardware_timers->try_append(hpet_comparator).release_value_but_fixme_should_propagate_errors(); auto periodic_timers = scan_and_initialize_periodic_timers(); auto non_periodic_timers = scan_for_non_periodic_timers(); @@ -447,10 +447,11 @@ UNMAP_AFTER_INIT bool TimeManagement::probe_and_set_x86_legacy_hardware_timers() } } - m_hardware_timers.append(PIT::initialize(TimeManagement::update_time)); - m_hardware_timers.append(RealTimeClock::create(TimeManagement::system_timer_tick)); - m_time_keeper_timer = m_hardware_timers[0]; - m_system_timer = m_hardware_timers[1]; + VERIFY(s_hardware_timers->is_empty()); + s_hardware_timers->try_append(PIT::initialize(TimeManagement::update_time)).release_value_but_fixme_should_propagate_errors(); + s_hardware_timers->try_append(RealTimeClock::create(TimeManagement::system_timer_tick)).release_value_but_fixme_should_propagate_errors(); + m_time_keeper_timer = (*s_hardware_timers)[0]; + m_system_timer = (*s_hardware_timers)[1]; // The timer is only as accurate as the interrupts... m_time_ticks_per_second = m_time_keeper_timer->ticks_per_second(); @@ -490,21 +491,11 @@ void TimeManagement::increment_time_since_boot_hpet() #elif ARCH(AARCH64) UNMAP_AFTER_INIT bool TimeManagement::probe_and_set_aarch64_hardware_timers() { - for (auto& recipe : *s_recipes) { - auto device_or_error = recipe.create_device(); - if (device_or_error.is_error()) { - dmesgln("TimeManagement: Failed to create timer for device \"{}\" with driver {}: {}", recipe.node_name, recipe.driver_name, device_or_error.release_error()); - continue; - } - - m_hardware_timers.append(device_or_error.release_value()); - } - - if (m_hardware_timers.is_empty()) - PANIC("TimeManagement: No supported timer found in devicetree"); + if (s_hardware_timers->is_empty()) + PANIC("TimeManagement: No supported timer was found"); // TODO: Use some kind of heuristic to decide which timer to use. - m_system_timer = m_hardware_timers.last(); + m_system_timer = s_hardware_timers->last(); dbgln("TimeManagement: System timer: {}", m_system_timer->model()); m_time_ticks_per_second = m_system_timer->ticks_per_second(); @@ -542,8 +533,8 @@ UNMAP_AFTER_INIT bool TimeManagement::probe_and_set_aarch64_hardware_timers() #elif ARCH(RISCV64) UNMAP_AFTER_INIT bool TimeManagement::probe_and_set_riscv64_hardware_timers() { - m_hardware_timers.append(RISCV64::Timer::initialize()); - m_system_timer = m_hardware_timers[0]; + MUST(s_hardware_timers->try_append(RISCV64::Timer::initialize())); + m_system_timer = (*s_hardware_timers)[0]; m_time_ticks_per_second = m_system_timer->ticks_per_second(); m_system_timer->set_callback([this]() { @@ -604,10 +595,7 @@ void TimeManagement::increment_time_since_boot() void TimeManagement::system_timer_tick() { - if (Processor::current_in_irq() <= 1) { - // Don't expire timers while handling IRQs - TimerQueue::the().fire(); - } + TimerQueue::the().fire(); Scheduler::timer_tick(); } diff --git a/Kernel/Time/TimeManagement.h b/Kernel/Time/TimeManagement.h index 805aca396be9dd..4a08c6a32ef5e7 100644 --- a/Kernel/Time/TimeManagement.h +++ b/Kernel/Time/TimeManagement.h @@ -14,7 +14,6 @@ #include #include #include -#include #include #include #include @@ -39,7 +38,7 @@ class TimeManagement { static bool is_initialized(); static TimeManagement& the(); - static void add_recipe(DeviceTree::DeviceRecipe>); + static ErrorOr register_hardware_timer(NonnullLockRefPtr); static u64 scheduler_current_time(); @@ -97,7 +96,6 @@ class TimeManagement { #endif Vector scan_and_initialize_periodic_timers(); Vector scan_for_non_periodic_timers(); - Vector> m_hardware_timers; void set_system_timer(HardwareTimerBase&); static void system_timer_tick(); diff --git a/Kernel/kprintf.cpp b/Kernel/kprintf.cpp index 234c3502b0e930..14383e68d542be 100644 --- a/Kernel/kprintf.cpp +++ b/Kernel/kprintf.cpp @@ -26,6 +26,7 @@ extern Atomic g_boot_console; } static bool s_serial_debug_enabled; +static bool s_pci_serial_debug_enabled; // A recursive spinlock allows us to keep writing in the case where a // page fault happens in the middle of a dbgln(), etc static RecursiveSpinlock s_log_lock {}; @@ -35,17 +36,30 @@ void set_serial_debug_enabled(bool desired_state) s_serial_debug_enabled = desired_state; } +void set_pci_serial_debug_enabled(bool desired_state) +{ + s_pci_serial_debug_enabled = desired_state; +} + bool is_serial_debug_enabled() { return s_serial_debug_enabled; } +bool is_pci_serial_debug_enabled() +{ + return s_pci_serial_debug_enabled; +} + static void serial_putch(char ch) +{ + debug_output(ch); +} + +static void pci_serial_putch(char ch) { if (PCISerial16550::is_available()) PCISerial16550::the().put_char(ch); - - debug_output(ch); } static void critical_console_out(char ch) @@ -53,6 +67,9 @@ static void critical_console_out(char ch) if (s_serial_debug_enabled) serial_putch(ch); + if (s_pci_serial_debug_enabled) + pci_serial_putch(ch); + #if ARCH(X86_64) // No need to output things to the real ConsoleDevice as no one is likely // to read it (because we are in a fatal situation, so only print things and halt) @@ -77,6 +94,9 @@ static void console_out(char ch) if (s_serial_debug_enabled) serial_putch(ch); + if (s_pci_serial_debug_enabled) + pci_serial_putch(ch); + #if ARCH(X86_64) bochs_debug_output(ch); #endif @@ -100,6 +120,10 @@ static inline void internal_dbgputch(char ch) { if (s_serial_debug_enabled) serial_putch(ch); + + if (s_pci_serial_debug_enabled) + pci_serial_putch(ch); + #if ARCH(X86_64) bochs_debug_output(ch); #endif diff --git a/Kernel/kstdio.h b/Kernel/kstdio.h index 03f6d65048ea3e..17d7492d722d90 100644 --- a/Kernel/kstdio.h +++ b/Kernel/kstdio.h @@ -16,7 +16,9 @@ void kernelcriticalputstr(char const*, size_t); void dbgputchar(char); void kernelearlyputstr(char const*, size_t); void set_serial_debug_enabled(bool desired_state); +void set_pci_serial_debug_enabled(bool desired_state); bool is_serial_debug_enabled(); +bool is_pci_serial_debug_enabled(); } void dbgputstr(StringView view); diff --git a/Ladybird/Qt/WebContentView.cpp b/Ladybird/Qt/WebContentView.cpp index ffdb640246b548..768a169a8c9ee9 100644 --- a/Ladybird/Qt/WebContentView.cpp +++ b/Ladybird/Qt/WebContentView.cpp @@ -796,8 +796,8 @@ bool WebContentView::event(QEvent* event) void WebContentView::enqueue_native_event(Web::MouseEvent::Type type, QSinglePointEvent const& event) { - Web::DevicePixelPoint position = { event.position().x() * m_device_pixel_ratio, event.position().y() * m_device_pixel_ratio }; - auto screen_position = Gfx::IntPoint { event.globalPosition().x() * m_device_pixel_ratio, event.globalPosition().y() * m_device_pixel_ratio }; + Web::DevicePixelPoint position = { event.position().x() * static_cast(m_device_pixel_ratio), event.position().y() * static_cast(m_device_pixel_ratio) }; + auto screen_position = Gfx::IntPoint { event.globalPosition().x() * static_cast(m_device_pixel_ratio), event.globalPosition().y() * static_cast(m_device_pixel_ratio) }; auto button = get_button_from_qt_mouse_button(event.button()); auto buttons = get_buttons_from_qt_mouse_buttons(event.buttons()); @@ -846,10 +846,10 @@ struct DragData : Web::ChromeInputData { void WebContentView::enqueue_native_event(Web::DragEvent::Type type, QDropEvent const& event) { - Web::DevicePixelPoint position = { event.position().x() * m_device_pixel_ratio, event.position().y() * m_device_pixel_ratio }; + Web::DevicePixelPoint position = { event.position().x() * static_cast(m_device_pixel_ratio), event.position().y() * static_cast(m_device_pixel_ratio) }; auto global_position = mapToGlobal(event.position()); - auto screen_position = Gfx::IntPoint { global_position.x() * m_device_pixel_ratio, global_position.y() * m_device_pixel_ratio }; + auto screen_position = Gfx::IntPoint { global_position.x() * static_cast(m_device_pixel_ratio), global_position.y() * static_cast(m_device_pixel_ratio) }; auto button = get_button_from_qt_mouse_button(Qt::LeftButton); auto buttons = get_buttons_from_qt_mouse_buttons(event.buttons()); diff --git a/Meta/CMake/ca_certificates_data.cmake b/Meta/CMake/ca_certificates_data.cmake index f8229a56eb8d18..7e145821726ff7 100644 --- a/Meta/CMake/ca_certificates_data.cmake +++ b/Meta/CMake/ca_certificates_data.cmake @@ -1,7 +1,7 @@ include(${CMAKE_CURRENT_LIST_DIR}/utils.cmake) -set(CACERT_VERSION "2025-11-04") -set(CACERT_SHA256 "8ac40bdd3d3e151a6b4078d2b2029796e8f843e3f86fbf2adbc4dd9f05e79def") +set(CACERT_VERSION "2025-12-02") +set(CACERT_SHA256 "f1407d974c5ed87d544bd931a278232e13925177e239fca370619aba63c757b4") set(CACERT_PATH "${SERENITY_CACHE_DIR}/CACERT" CACHE PATH "Download location for cacert.pem") set(CACERT_VERSION_FILE "${CACERT_PATH}/version.txt") diff --git a/Meta/CMake/common_compile_options.cmake b/Meta/CMake/common_compile_options.cmake index a1d3b6f7d4820f..1493174894e989 100644 --- a/Meta/CMake/common_compile_options.cmake +++ b/Meta/CMake/common_compile_options.cmake @@ -64,6 +64,9 @@ if (CMAKE_CXX_COMPILER_ID MATCHES "Clang$") add_compile_options(-Wno-vla-cxx-extension) add_compile_options(-Wno-coroutine-missing-unhandled-exception) elseif (CMAKE_CXX_COMPILER_ID STREQUAL "GNU") + add_compile_options(-Wcast-align) + add_compile_options(-Wdouble-promotion) + # Only ignore expansion-to-defined for g++, clang's implementation doesn't complain about function-like macros add_compile_options(-Wno-expansion-to-defined) add_compile_options(-Wno-literal-suffix) diff --git a/Meta/CMake/serenity_compile_options.cmake b/Meta/CMake/serenity_compile_options.cmake index 364bf1f7877b65..e04d1120e9e590 100644 --- a/Meta/CMake/serenity_compile_options.cmake +++ b/Meta/CMake/serenity_compile_options.cmake @@ -19,8 +19,6 @@ add_link_options(LINKER:-Bsymbolic-non-weak-functions) if (CMAKE_CXX_COMPILER_ID STREQUAL "GNU") add_compile_options(-Wno-maybe-uninitialized) - add_compile_options(-Wcast-align) - add_compile_options(-Wdouble-promotion) elseif (CMAKE_CXX_COMPILER_ID MATCHES "Clang$") add_compile_options(-Wno-atomic-alignment) add_compile_options(-Wno-unused-const-variable) diff --git a/Meta/Lagom/CMakeLists.txt b/Meta/Lagom/CMakeLists.txt index ee258d1b27af77..ea41020b52d183 100644 --- a/Meta/Lagom/CMakeLists.txt +++ b/Meta/Lagom/CMakeLists.txt @@ -683,26 +683,27 @@ if (BUILD_LAGOM) # LibTest tests from Tests/ set(TEST_DIRECTORIES AK - LibCrypto LibCompress + LibCrypto LibDisassembly LibGL LibGfx LibHID LibHTTP LibIMAP + LibLine LibLocale LibMarkdown + LibMedia LibPDF LibSQL - LibMedia - LibTest - LibTextCodec LibTTF + LibTest + LibTextCodec LibTimeZone - LibUnicode LibURL - LibXML + LibUnicode + LibXML ) if (ENABLE_LAGOM_LIBWEB) list(APPEND TEST_DIRECTORIES LibWeb) diff --git a/Meta/analyze-qemu-coverage.sh b/Meta/analyze-qemu-coverage.sh index 723a5f7f53cb64..0019483b8ac682 100755 --- a/Meta/analyze-qemu-coverage.sh +++ b/Meta/analyze-qemu-coverage.sh @@ -44,9 +44,9 @@ mapfile -d '\n' all_binaries < <(find "$BUILD_DIR"/Root -type f -exec sh -c "fil COVERAGE_PREPARE="$BUILD_DIR/prepare-code-coverage-artifact.py" if [ ! -f "$COVERAGE_PREPARE" ]; then # Download coverage prep script from github - LLVM_14_RELEASE_HASH=329fda39c507e8740978d10458451dcdb21563be - SHA256_SUM=2cf1019d1df9a10c87234e0ec9c984dbb97d5543688b7f4a7387cb377ced7f21 - URL=https://raw.githubusercontent.com/llvm/llvm-project/${LLVM_14_RELEASE_HASH}/llvm/utils/prepare-code-coverage-artifact.py + LLVM_GIT_VERSION=9decb102d9a1e7dc55883a633789cb2563de2b25 + SHA256_SUM=df159c0c9d8129505688ccb42d6066fa36e1bab6e3233417d6c3d26d21d40a5f + URL=https://raw.githubusercontent.com/llvm/llvm-project/${LLVM_GIT_VERSION}/llvm/utils/prepare-code-coverage-artifact.py echo "Downloading prepare-code-coverage-artifact.py from ${URL}" wget "$URL" -P "$BUILD_DIR" diff --git a/Meta/build-image-qemu.sh b/Meta/build-image-qemu.sh index 7dfc4d24195551..e2c19144d88c3d 100755 --- a/Meta/build-image-qemu.sh +++ b/Meta/build-image-qemu.sh @@ -39,6 +39,10 @@ INODE_COUNT=$((INODE_COUNT + 2000)) # Some additional inodes for toolchain file DISK_SIZE_BYTES=$((($(disk_usage "$SERENITY_SOURCE_DIR/Base") + $(disk_usage Root) ) * 1024 * 1024)) DISK_SIZE_BYTES=$((DISK_SIZE_BYTES + (INODE_COUNT * INODE_SIZE))) +# At the time of writing, in the default path DISK_SIZE_BYTES ~= 1.4 GiB and INODE_COUNT ~= 140k. +# Let's use this ratio of bytes per inode instead of manually trying to guess the number of inodes. +BYTES_PER_INODE=11264 + if [ -z "$SERENITY_DISK_SIZE_BYTES" ]; then # Try to use heuristics to guess a good disk size and inode count. # The disk must notably fit: @@ -47,7 +51,6 @@ if [ -z "$SERENITY_DISK_SIZE_BYTES" ]; then # * Inodes and block bitmaps for each block group, # * Plenty of extra free space and free inodes. DISK_SIZE_BYTES=$((DISK_SIZE_BYTES * 2)) - INODE_COUNT=$((INODE_COUNT * 7)) else if [ "$DISK_SIZE_BYTES" -gt "$SERENITY_DISK_SIZE_BYTES" ]; then die "SERENITY_DISK_SIZE_BYTES is set to $SERENITY_DISK_SIZE_BYTES but required disk size is $DISK_SIZE_BYTES bytes" @@ -55,13 +58,6 @@ else DISK_SIZE_BYTES="$SERENITY_DISK_SIZE_BYTES" fi -if [ -n "$SERENITY_INODE_COUNT" ]; then - if [ "$INODE_COUNT" -gt "$SERENITY_INODE_COUNT" ]; then - die "SERENITY_INODE_COUNT is set to $SERENITY_INODE_COUNT but required inode count is roughly $INODE_COUNT" - fi - INODE_COUNT="$SERENITY_INODE_COUNT" -fi - nearest_power_of_2() { local n=$1 local p=1 @@ -116,9 +112,9 @@ if [ $USE_EXISTING -ne 1 ]; then if [ "$(uname -s)" = "OpenBSD" ]; then VND=$(vnconfig _disk_image) (echo "e 0"; echo 83; echo n; echo 0; echo "*"; echo "quit") | fdisk -e "$VND" - newfs_ext2fs -D "${INODE_SIZE}" -n "${INODE_COUNT}" "/dev/r${VND}i" || die "could not create filesystem" + newfs_ext2fs -D "${INODE_SIZE}" -i "${BYTES_PER_INODE}" "/dev/r${VND}i" || die "could not create filesystem" else - "${MKE2FS_PATH}" -q -I "${INODE_SIZE}" -N "${INODE_COUNT}" _disk_image || die "could not create filesystem" + "${MKE2FS_PATH}" -q -I "${INODE_SIZE}" -i "${BYTES_PER_INODE}" _disk_image || die "could not create filesystem" fi echo "done" fi @@ -176,7 +172,7 @@ script_path=$(cd -P -- "$(dirname -- "$0")" && pwd -P) "$script_path/build-root-filesystem.sh" if [ $use_genext2fs = 1 ]; then - genext2fs -B 4096 -b $((DISK_SIZE_BYTES / 4096)) -N "${INODE_COUNT}" -d mnt _disk_image || die "try increasing image size (genext2fs -b)" + genext2fs -B 4096 -b $((DISK_SIZE_BYTES / 4096)) -i "${BYTES_PER_INODE}" -d mnt _disk_image || die "try increasing image size (genext2fs -b)" # if using docker with shared mount, file is created as root, so make it writable for users chmod 0666 _disk_image fi diff --git a/Meta/check-jbig2-json.sh b/Meta/check-jbig2-json.sh index 6f3a50197bc8c3..f54b74c36f6a7d 100755 --- a/Meta/check-jbig2-json.sh +++ b/Meta/check-jbig2-json.sh @@ -2,6 +2,19 @@ set -eo pipefail +jbig_files=(Tests/LibGfx/test-inputs/jbig2/*.jbig2) +json_files=(Tests/LibGfx/test-inputs/jbig2/json/*.json) + +if [ "${#jbig_files[@]}" -gt "${#json_files[@]}" ]; then + echo "More jbig2 than json files. Don't add non-json-based jbig2 files." + exit 1 +fi + +if [ "${#jbig_files[@]}" -lt "${#json_files[@]}" ]; then + echo "More json than jbig2 files. Did you forget to 'git add'?" + exit 1 +fi + trap 'git diff --exit-code' EXIT script_path=$(cd -P -- "$(dirname -- "$0")" && pwd -P) @@ -20,10 +33,7 @@ if [ -z "${JBIG2_FROM_JSON_BINARY:-}" ] ; then JBIG2_FROM_JSON_BINARY="Build/lagom/bin/jbig2-from-json" fi -for f in Tests/LibGfx/test-inputs/jbig2/json/*.json; do - f_jb2=Tests/LibGfx/test-inputs/jbig2/$(basename "${f%.json}.jbig2") - "$JBIG2_FROM_JSON_BINARY" -o "$f_jb2" "$f" -done +Tests/LibGfx/test-inputs/jbig2/json/compile.sh "$JBIG2_FROM_JSON_BINARY" # annex-h.jbig2 is fixed data from Annex H of the JBIG2 spec. # It should never change, if it does, that's a bug. diff --git a/Meta/find_compiler.sh b/Meta/find_compiler.sh index f7ae5920412df4..40cc91473947ed 100644 --- a/Meta/find_compiler.sh +++ b/Meta/find_compiler.sh @@ -56,7 +56,7 @@ pick_host_compiler() { return fi - find_newest_compiler clang clang-17 clang-18 /opt/homebrew/opt/llvm/bin/clang + find_newest_compiler clang clang-{17..21} /opt/homebrew/opt/llvm/bin/clang if is_supported_compiler "$HOST_COMPILER"; then export CC="${HOST_COMPILER}" export CXX="${HOST_COMPILER/clang/clang++}" diff --git a/Meta/gn/secondary/Userland/Libraries/LibTLS/BUILD.gn b/Meta/gn/secondary/Userland/Libraries/LibTLS/BUILD.gn index b73bb27e62a618..e719927a639cf0 100644 --- a/Meta/gn/secondary/Userland/Libraries/LibTLS/BUILD.gn +++ b/Meta/gn/secondary/Userland/Libraries/LibTLS/BUILD.gn @@ -9,7 +9,7 @@ declare_args() { cacert_cache = cache_path + "CACERT/" -cacert_version = "2025-11-04" +cacert_version = "2025-12-02" if (enable_cacert_download) { download_file("ca_certificates_download") { @@ -17,7 +17,7 @@ if (enable_cacert_download) { url = "https://curl.se/ca/cacert-$version.pem" output = "$root_build_dir/cacert.pem" version_file = cacert_cache + "version.txt" - sha256 = "8ac40bdd3d3e151a6b4078d2b2029796e8f843e3f86fbf2adbc4dd9f05e79def" + sha256 = "f1407d974c5ed87d544bd931a278232e13925177e239fca370619aba63c757b4" } # FIXME: Copy file to /etc/cacert.pem on serenity } diff --git a/Meta/jbig2_to_pdf.py b/Meta/jbig2_to_pdf.py index 176722f40c8fa2..f3f860ab45ebf6 100755 --- a/Meta/jbig2_to_pdf.py +++ b/Meta/jbig2_to_pdf.py @@ -190,7 +190,6 @@ def main(): p = 4 if global_segments else 3 - print(f'{len(pages)} pages') page_refs = b' '.join([b'%d 0 R' % (p + 3 * i) for i in range(len(pages))]) global_entry = b'' @@ -235,7 +234,6 @@ def main(): for page in pages: segment_headers = pages[page] width, height = get_dimensions(segment_headers) - print(f'dims {width}x{height}') segment_headers = [h for h in segment_headers if h.type != EndOfPage] image_data = reserialize(segment_headers) diff --git a/Meta/run.py b/Meta/run.py index dd2889c522165d..d30678b37d30b4 100755 --- a/Meta/run.py +++ b/Meta/run.py @@ -761,40 +761,42 @@ def set_up_machine_devices(config: Configuration): config.vga_type = None config.display_device = None config.kernel_cmdline.append("serial_debug") - if config.machine_type != MachineType.CI: - # FIXME: Windows QEMU crashes when we set the same display as usual here. - config.display_backend = None - config.audio_devices = [] - caches_path = BUILD_DIRECTORY.parent / "caches" - dtb_path = str(caches_path / "bcm2710-rpi-3-b.dtb") - if config.machine_type == MachineType.RaspberryPi4B: - dtb_path = str(caches_path / "bcm2711-rpi-4-b.dtb") + # FIXME: Windows QEMU crashes when we set the same display as usual here. + config.display_backend = None + config.audio_devices = [] - config.extra_arguments.extend( - [ - "-serial", "stdio", - "-dtb", dtb_path - ] - ) - config.qemu_cpu = None - return + caches_path = BUILD_DIRECTORY.parent / "caches" + dtb_path = str(caches_path / "bcm2710-rpi-3-b.dtb") + if config.machine_type == MachineType.RaspberryPi4B: + dtb_path = str(caches_path / "bcm2711-rpi-4-b.dtb") + + config.extra_arguments.extend( + [ + "-serial", "stdio", + "-dtb", dtb_path + ] + ) + config.qemu_cpu = None + return elif config.architecture == Arch.Aarch64 or config.architecture == Arch.RISCV64: config.qemu_machine = "virt" config.cpu_count = None config.audio_devices = [] - config.extra_arguments.extend(["-serial", "stdio"]) config.kernel_cmdline.extend(["serial_debug"]) config.qemu_cpu = "max" if config.architecture == Arch.Aarch64 else None - config.add_devices( - [ - "virtio-keyboard", - "virtio-tablet", - "virtio-serial,max_ports=2", - ] - ) - return + + if config.machine_type != MachineType.CI: + config.extra_arguments.extend(["-serial", "stdio"]) + config.add_devices( + [ + "virtio-keyboard", + "virtio-tablet", + "virtio-serial,max_ports=2", + ] + ) + return # Machine specific base setups if config.machine_type in [MachineType.QEMU35Grub, MachineType.QEMU35]: @@ -835,13 +837,14 @@ def set_up_machine_devices(config: Configuration): config.display_backend = "none" config.audio_backend = None config.audio_devices = [] - config.extra_arguments.extend(["-serial", "stdio", "-no-reboot", "-monitor", "none"]) + config.extra_arguments.extend(["-no-reboot", "-monitor", "none"]) config.spice_arguments = [] - if config.architecture == Arch.Aarch64: - config.extra_arguments.extend(["-serial", "file:debug.log"]) + if config.architecture == Arch.Aarch64 or config.architecture == Arch.RISCV64: + config.character_devices.append("stdio,id=stdout") + config.extra_arguments.extend(["-device", "pci-serial,chardev=stdout", "-serial", "file:debug.log"]) else: config.add_device("ich9-ahci") - config.extra_arguments.extend(["-debugcon", "file:debug.log"]) + config.extra_arguments.extend(["-serial", "stdio", "-debugcon", "file:debug.log"]) else: # Default machine diff --git a/Meta/shell_include.sh b/Meta/shell_include.sh index 15f99aaf990c2f..cdea0c5b89f883 100644 --- a/Meta/shell_include.sh +++ b/Meta/shell_include.sh @@ -37,7 +37,7 @@ find_executable() { paths=("/usr/sbin" "/sbin") if [ "$(uname -s)" = "Darwin" ]; then - if [ -n "${HOMEBREW_PREFIX}" ]; then + if [ -n "${HOMEBREW_PREFIX:-}" ]; then paths+=("${HOMEBREW_PREFIX}/opt/e2fsprogs/bin" "${HOMEBREW_PREFIX}/opt/e2fsprogs/sbin") elif command -v brew > /dev/null 2>&1; then if prefix=$(brew --prefix e2fsprogs 2>/dev/null); then diff --git a/Meta/tweet-commits.js b/Meta/tweet-commits.js deleted file mode 100644 index cb601f7f9b108a..00000000000000 --- a/Meta/tweet-commits.js +++ /dev/null @@ -1,35 +0,0 @@ -const fs = require("fs"); -const Twit = require("twit"); -const { CONSUMER_KEY, CONSUMER_SECRET, ACCESS_TOKEN, ACCESS_TOKEN_SECRET } = process.env; -const tweetLength = 280; -// Twitter always considers t.co links to be 23 chars, see https://help.twitter.com/en/using-twitter/how-to-tweet-a-link -const twitterLinkLength = 23; - -const T = new Twit({ - consumer_key: CONSUMER_KEY, - consumer_secret: CONSUMER_SECRET, - access_token: ACCESS_TOKEN, - access_token_secret: ACCESS_TOKEN_SECRET, -}); - -(async () => { - const githubEvent = JSON.parse(fs.readFileSync(0).toString()); - const tweets = []; - for (const commit of githubEvent["commits"]) { - const authorLine = `Author: ${commit["author"]["name"]}`; - const maxMessageLength = tweetLength - authorLine.length - twitterLinkLength - 2; // -2 for newlines - const commitMessage = - commit["message"].length > maxMessageLength - ? commit["message"].substring(0, maxMessageLength - 2) + "…" // Ellipsis counts as 2 characters - : commit["message"]; - - tweets.push(`${commitMessage}\n${authorLine}\n${commit["url"]}`); - } - for (const tweet of tweets) { - try { - await T.post("statuses/update", { status: tweet }); - } catch (e) { - console.error("Failed to post a tweet!", e.message); - } - } -})(); diff --git a/Ports/AvailablePorts.md b/Ports/AvailablePorts.md index 86a3b3e33188af..2b991305376f3a 100644 --- a/Ports/AvailablePorts.md +++ b/Ports/AvailablePorts.md @@ -30,7 +30,7 @@ This list is also available at [ports.serenityos.net](https://ports.serenityos.n | [`bzip3`](bzip3/) | bzip3 | 1.5.1 | https://github.com/kspalaiologos/bzip3 | | [`c-ares`](c-ares/) | c-ares | 1.19.0 | https://c-ares.org | | [`c-ray`](c-ray/) | C-Ray | 8f30eb9 | https://github.com/vkoskiv/c-ray | -| [`ca-certificates`](ca-certificates/) | Mozilla CA certificate store | 2025-11-04 | https://curl.se/docs/caextract.html | +| [`ca-certificates`](ca-certificates/) | Mozilla CA certificate store | 2025-12-02 | https://curl.se/docs/caextract.html | | [`cairo`](cairo/) | Cairo | 1.18.2 | https://www.cairographics.org/ | | [`carl`](carl/) | Crypto Ancienne Resource Loader | 1.5 | https://github.com/classilla/cryanc | | [`cavestory`](cavestory/) | Cave Story | 2.6.5-1 | https://github.com/nxengine/nxengine-evo | @@ -45,7 +45,7 @@ This list is also available at [ports.serenityos.net](https://ports.serenityos.n | [`cmake`](cmake/) | CMake | 3.26.4 | https://cmake.org/ | | [`cmatrix`](cmatrix/) | cmatrix | 5c082c6 | https://github.com/abishekvashok/cmatrix | | [`composer`](composer/) | Composer | 2.6.5 | https://getcomposer.org/ | -| [`coreutils`](coreutils/) | GNU core utilities | 9.5 | https://www.gnu.org/software/coreutils/ | +| [`coreutils`](coreutils/) | GNU core utilities | 9.9 | https://www.gnu.org/software/coreutils/ | | [`cowsay`](cowsay/) | cowsay | 3.04 | https://github.com/tnalpgge/rank-amateur-cowsay | | [`cpio`](cpio/) | GNU cpio archive utility | 2.15 | https://www.gnu.org/software/cpio/ | | [`curl`](curl/) | curl | 8.17.0 | https://curl.se/ | @@ -145,7 +145,7 @@ This list is also available at [ports.serenityos.net](https://ports.serenityos.n | [`less`](less/) | less | 643 | https://www.greenwoodsoftware.com/less/ | | [`libarchive`](libarchive/) | libarchive | 3.7.7 | https://libarchive.org/ | | [`libassuan`](libassuan/) | libassuan | 2.5.7 | https://gnupg.org/software/libassuan/index.html | -| [`libatomic_ops`](libatomic_ops/) | libatomic_ops | 7.8.2 | https://www.hboehm.info/gc/ | +| [`libatomic_ops`](libatomic_ops/) | libatomic_ops | 7.10.0 | https://github.com/bdwgc/libatomic_ops/ | | [`libenet`](libenet/) | libenet | 1.3.17 | http://sauerbraten.org/enet/ | | [`libexpat`](libexpat/) | Expat | 2.6.4 | https://libexpat.github.io/ | | [`libffi`](libffi/) | libffi | 3.4.5 | https://www.sourceware.org/libffi/ | @@ -227,7 +227,7 @@ This list is also available at [ports.serenityos.net](https://ports.serenityos.n | [`msttcorefonts`](msttcorefonts/) | Microsoft's TrueType core fonts | | https://corefonts.sourceforge.net/ | | [`mysthous`](mysthous/) | Hi-Res Adventure #1: Mystery House | 1.0 | https://www.scummvm.org/games/#games-hires1 | | [`nano`](nano/) | GNU nano | 8.5 | https://www.nano-editor.org/ | -| [`nasm`](nasm/) | Netwide Assembler (NASM) | 2.16.03 | https://www.nasm.us/ | +| [`nasm`](nasm/) | Netwide Assembler (NASM) | 3.01 | https://www.nasm.us/ | | [`ncdu`](ncdu/) | Ncdu | 1.18.1 | https://dev.yorhel.nl/ncdu | | [`ncurses`](ncurses/) | ncurses | 6.5 | https://invisible-island.net/ncurses/announce.html | | [`neofetch`](neofetch/) | neofetch | 7.1.0 | https://github.com/dylanaraps/neofetch | @@ -266,7 +266,7 @@ This list is also available at [ports.serenityos.net](https://ports.serenityos.n | [`pcre2`](pcre2/) | Perl-compatible Regular Expressions (PCRE2) | 10.45 | https://www.pcre.org/ | | [`perl5`](perl5/) | Perl | 5.40.1 | https://www.perl.org/ | | [`pfetch`](pfetch/) | pfetch | a906ff8 | https://github.com/dylanaraps/pfetch/ | -| [`php`](php/) | PHP | 8.2.10 | https://www.php.net/ | +| [`php`](php/) | PHP | 8.5.0 | https://www.php.net/ | | [`pixman`](pixman/) | pixman | 0.42.2 | http://pixman.org | | [`pkgconf`](pkgconf/) | pkgconf | 2.0.2 | https://github.com/pkgconf/pkgconf | | [`poppler`](poppler/) | Poppler is a PDF rendering library | 24.02.0 | https://poppler.freedesktop.org/ | @@ -355,7 +355,7 @@ This list is also available at [ports.serenityos.net](https://ports.serenityos.n | [`xash3d-fwgs`](xash3d-fwgs/) | Xash3D FWGS game engine | 2022.12.26 | https://github.com/FWGS/xash3d-fwgs | | [`xmp-cli`](xmp-cli/) | Extended Module Player | 4.2.0 | https://github.com/libxmp/xmp-cli | | [`xorriso`](xorriso/) | xorriso | 1.5.6 | https://www.gnu.org/software/xorriso | -| [`xz`](xz/) | xz | 5.6.2 | https://tukaani.org/xz/ | +| [`xz`](xz/) | xz | 5.8.1 | https://tukaani.org/xz/ | | [`yasm`](yasm/) | Yasm Modular Assembler | 1.3.0 | https://yasm.tortall.net/ | | [`zig`](zig/) | Zig programming language | 0.16.0-dev.627+e6e4792a5 | https://ziglang.org/ | | [`zlib`](zlib/) | zlib | 1.3.1 | https://www.zlib.net/ | diff --git a/Ports/ca-certificates/package.sh b/Ports/ca-certificates/package.sh index 20695721bf3a86..c1dc96376d7067 100755 --- a/Ports/ca-certificates/package.sh +++ b/Ports/ca-certificates/package.sh @@ -1,8 +1,8 @@ #!/usr/bin/env -S bash ../.port_include.sh port='ca-certificates' -version='2025-11-04' +version='2025-12-02' files=( - "https://curl.se/ca/cacert-${version}.pem#8ac40bdd3d3e151a6b4078d2b2029796e8f843e3f86fbf2adbc4dd9f05e79def" + "https://curl.se/ca/cacert-${version}.pem#f1407d974c5ed87d544bd931a278232e13925177e239fca370619aba63c757b4" ) workdir='.' diff --git a/Ports/coreutils/package.sh b/Ports/coreutils/package.sh index 1c61ddc1ba7e8d..2993f2f2b82b4e 100755 --- a/Ports/coreutils/package.sh +++ b/Ports/coreutils/package.sh @@ -1,13 +1,9 @@ #!/usr/bin/env -S bash ../.port_include.sh port='coreutils' -version='9.5' +version='9.9' useconfigure='true' -use_fresh_config_sub='true' -config_sub_paths=( - 'build-aux/config.sub' -) files=( - "https://ftpmirror.gnu.org/gnu/coreutils/coreutils-${version}.tar.gz#767ae6a22950ec42f3ba5f7c1de79dd27800ee8e9b8642da5dedb5974a1741e5" + "https://ftpmirror.gnu.org/gnu/coreutils/coreutils-${version}.tar.gz#91a719fcf923de686016f2c8d084a8be1f793f34173861273c4668f7c65af94a" ) # Exclude some non-working utilities: diff --git a/Ports/libatomic_ops/package.sh b/Ports/libatomic_ops/package.sh index ee2cd84a370a6c..6a480b8efadfd6 100755 --- a/Ports/libatomic_ops/package.sh +++ b/Ports/libatomic_ops/package.sh @@ -1,8 +1,8 @@ #!/usr/bin/env -S bash ../.port_include.sh port='libatomic_ops' -version='7.8.2' +version='7.10.0' useconfigure='true' use_fresh_config_sub='true' files=( - "https://github.com/ivmai/libatomic_ops/releases/download/v${version}/libatomic_ops-${version}.tar.gz#d305207fe207f2b3fb5cb4c019da12b44ce3fcbc593dfd5080d867b1a2419b51" + "https://github.com/bdwgc/libatomic_ops/releases/download/v${version}/libatomic_ops-${version}.tar.gz#0db3ebff755db170f65e74a64ec4511812e9ee3185c232eeffeacd274190dfb0" ) diff --git a/Ports/nasm/package.sh b/Ports/nasm/package.sh index 66e1f092c9660c..5ab7911768f313 100755 --- a/Ports/nasm/package.sh +++ b/Ports/nasm/package.sh @@ -1,10 +1,8 @@ #!/usr/bin/env -S bash ../.port_include.sh -port=nasm -version=2.16.03 +port='nasm' +version='3.01' files=( - "https://www.nasm.us/pub/nasm/releasebuilds/${version}/nasm-${version}.tar.gz#5bc940dd8a4245686976a8f7e96ba9340a0915f2d5b88356874890e207bdb581" + "https://www.nasm.us/pub/nasm/releasebuilds/${version}/nasm-${version}.tar.gz#aea120d4adb0241f08ae24d6add09e4a993bc1c4d9f754dbfc8020d6916c9be1" ) useconfigure=true -use_fresh_config_sub=true -config_sub_paths=("autoconf/helpers/config.sub") makeopts=() diff --git a/Ports/openssh/patches/0008-pledge-sigaction-in-scp.patch b/Ports/openssh/patches/0008-pledge-sigaction-in-scp.patch new file mode 100644 index 00000000000000..dfea8883637385 --- /dev/null +++ b/Ports/openssh/patches/0008-pledge-sigaction-in-scp.patch @@ -0,0 +1,22 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: Oskar Skog +Date: Sun, 14 Dec 2025 12:20:40 +0200 +Subject: [PATCH] pledge sigaction in scp + +--- + scp.c | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +diff --git a/scp.c b/scp.c +index c36d66aa59416d72495fdaecf881c902cb281e5b..b13a74a01e0f9bcd5190788a487ca5e586631a62 100644 +--- a/scp.c ++++ b/scp.c +@@ -606,7 +606,7 @@ main(int argc, char **argv) + if (pflag) { + /* Cannot pledge: -p allows setuid/setgid files... */ + } else { +- if (pledge("stdio rpath wpath cpath fattr tty proc exec", ++ if (pledge("stdio rpath wpath cpath fattr tty proc exec sigaction", + NULL) == -1) { + perror("pledge"); + exit(1); diff --git a/Ports/openssh/patches/ReadMe.md b/Ports/openssh/patches/ReadMe.md index 9bf1d180356c2b..8607c685bc0be8 100644 --- a/Ports/openssh/patches/ReadMe.md +++ b/Ports/openssh/patches/ReadMe.md @@ -35,3 +35,8 @@ Use sendfd/recvfd on serenity Use unveil for privsep +## `0008-pledge-sigaction-in-scp.patch` + +pledge sigaction in scp + + diff --git a/Ports/php/package.sh b/Ports/php/package.sh index fe136d21399481..3e99191b4bc2c1 100755 --- a/Ports/php/package.sh +++ b/Ports/php/package.sh @@ -1,9 +1,9 @@ #!/usr/bin/env -S bash ../.port_include.sh port='php' useconfigure='true' -version='8.2.10' +version='8.5.0' files=( - "https://www.php.net/distributions/php-${version}.tar.xz#561dc4acd5386e47f25be76f2c8df6ae854756469159248313bcf276e282fbb3" + "https://www.php.net/distributions/php-${version}.tar.xz#39cb6e4acd679b574d3d3276f148213e935fc25f90403eb84fb1b836a806ef1e" ) depends=( 'curl' @@ -16,7 +16,7 @@ depends=( ) configopts=( '--disable-cgi' - '--disable-opcache' + '--disable-opcache-jit' '--enable-fpm' "--prefix=${SERENITY_INSTALL_ROOT}/usr/local" '--with-curl' diff --git a/Ports/php/patches/0001-Build-Disable-pharcmd.patch b/Ports/php/patches/0001-Build-Disable-pharcmd.patch index 4fadd64020f748..f7c977d619b73e 100644 --- a/Ports/php/patches/0001-Build-Disable-pharcmd.patch +++ b/Ports/php/patches/0001-Build-Disable-pharcmd.patch @@ -10,10 +10,10 @@ not try to run phar locally. 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/configure.ac b/configure.ac -index 42d6a3b6ca968b488c871f7c6f5395530481138b..dd244cdbc28e650237249ae8b636f6034a578826 100644 +index a6078866df99986471679074841f679fe7332cd9..f73a3ab74762ba1e7c6878d1a4b35e889903f302 100644 --- a/configure.ac +++ b/configure.ac -@@ -1639,8 +1639,8 @@ CFLAGS="\$(CFLAGS_CLEAN) $standard_libtool_flag" +@@ -1614,8 +1614,8 @@ CFLAGS="\$(CFLAGS_CLEAN) $standard_libtool_flag" CXXFLAGS="$CXXFLAGS $standard_libtool_flag \$(PROF_FLAGS)" if test "$PHP_PHAR" != "no" && test "$PHP_CLI" != "no"; then diff --git a/Ports/php/patches/0002-Build-Force-inet_aton-detection.patch b/Ports/php/patches/0002-Build-Force-inet_aton-detection.patch deleted file mode 100644 index fd85b92b7fb32c..00000000000000 --- a/Ports/php/patches/0002-Build-Force-inet_aton-detection.patch +++ /dev/null @@ -1,26 +0,0 @@ -From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 -From: Jelle Raaijmakers -Date: Tue, 29 Mar 2022 22:42:18 +0200 -Subject: [PATCH] Build: Force `inet_aton` detection - -For a reason unknown to me, the build system fails to find `inet_aton` -and tries to redefine it with its own implementation in -`flock_compat.c`. ---- - configure.ac | 3 +-- - 1 file changed, 1 insertion(+), 2 deletions(-) - -diff --git a/configure.ac b/configure.ac -index dd244cdbc28e650237249ae8b636f6034a578826..1cbb6027c42d09b8fe0bc6f69afd023528795d74 100644 ---- a/configure.ac -+++ b/configure.ac -@@ -383,8 +383,7 @@ case $host_alias in - ;; - esac - --dnl Check for inet_aton in -lc, -lbind and -lresolv. --PHP_CHECK_FUNC(inet_aton, resolv, bind) -+AC_DEFINE(HAVE_INET_ATON, 1, [ ]) - - dnl Then headers. - dnl ---------------------------------------------------------------------------- diff --git a/Ports/php/patches/0003-Build-Patch-Serenity-root-directory-into-libtool.patch b/Ports/php/patches/0002-Build-Patch-Serenity-root-directory-into-libtool.patch similarity index 92% rename from Ports/php/patches/0003-Build-Patch-Serenity-root-directory-into-libtool.patch rename to Ports/php/patches/0002-Build-Patch-Serenity-root-directory-into-libtool.patch index 7c7399979744ce..b3df34b6f9626b 100644 --- a/Ports/php/patches/0003-Build-Patch-Serenity-root-directory-into-libtool.patch +++ b/Ports/php/patches/0002-Build-Patch-Serenity-root-directory-into-libtool.patch @@ -10,7 +10,7 @@ to get PHP to build. 1 file changed, 4 insertions(+) diff --git a/build/ltmain.sh b/build/ltmain.sh -index 2f1c8c9dc80f095832c58aeb30ca65fc27e8a64b..cce4f76267a211775ec97eb2b006df536cd4cc7b 100755 +index 3c00e79157079f42d54032c4096be1de5cb9fffe..e15618374d55082081188e36e3a6b442daf43b48 100755 --- a/build/ltmain.sh +++ b/build/ltmain.sh @@ -2353,6 +2353,9 @@ EOF diff --git a/Ports/php/patches/0003-Remove-include-of-sys-ipc.h.patch b/Ports/php/patches/0003-Remove-include-of-sys-ipc.h.patch new file mode 100644 index 00000000000000..08349f21167470 --- /dev/null +++ b/Ports/php/patches/0003-Remove-include-of-sys-ipc.h.patch @@ -0,0 +1,21 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: Linus Groh +Date: Sat, 6 Dec 2025 17:11:07 +0000 +Subject: [PATCH] Remove include of sys/ipc.h + +--- + ext/opcache/ZendAccelerator.c | 1 - + 1 file changed, 1 deletion(-) + +diff --git a/ext/opcache/ZendAccelerator.c b/ext/opcache/ZendAccelerator.c +index 84acae980ce3a5ec1d03e4aebe2f30e3fd601bb3..904a9c2ae08fd9acab89e0f192cde9fc56dc53e9 100644 +--- a/ext/opcache/ZendAccelerator.c ++++ b/ext/opcache/ZendAccelerator.c +@@ -89,7 +89,6 @@ typedef int gid_t; + #ifndef ZEND_WIN32 + # include + # include +-# include + # include + # include + #endif diff --git a/Ports/php/patches/0004-Disable-unsupported-prctl-call.patch b/Ports/php/patches/0004-Disable-unsupported-prctl-call.patch new file mode 100644 index 00000000000000..d8753b4c0630bb --- /dev/null +++ b/Ports/php/patches/0004-Disable-unsupported-prctl-call.patch @@ -0,0 +1,23 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: Linus Groh +Date: Sat, 6 Dec 2025 17:16:03 +0000 +Subject: [PATCH] Disable unsupported prctl call + +Serenity has prctl but doesn't define PR_SET_PDEATHSIG. +--- + sapi/cli/php_cli_server.c | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +diff --git a/sapi/cli/php_cli_server.c b/sapi/cli/php_cli_server.c +index da6732f06e72a297315615352a8f399107b7e096..21a4e3224bc4cd27045c346fe1334c01adae1019 100644 +--- a/sapi/cli/php_cli_server.c ++++ b/sapi/cli/php_cli_server.c +@@ -2486,7 +2486,7 @@ static char *php_cli_server_parse_addr(const char *addr, int *pport) { + static void php_cli_server_worker_install_pdeathsig(void) + { + // Ignore failure to register PDEATHSIG, it's not available on all platforms anyway +-#if defined(HAVE_PRCTL) ++#if defined(HAVE_PRCTL) && !defined(__serenity__) + prctl(PR_SET_PDEATHSIG, SIGTERM); + #elif defined(HAVE_PROCCTL) + int signal = SIGTERM; diff --git a/Ports/php/patches/ReadMe.md b/Ports/php/patches/ReadMe.md index 527b1dc59c40c9..b3cc2be2fdc679 100644 --- a/Ports/php/patches/ReadMe.md +++ b/Ports/php/patches/ReadMe.md @@ -7,18 +7,21 @@ Build: Disable `pharcmd` We do not support running the PHP binary locally after its build, so do not try to run phar locally. -## `0002-Build-Force-inet_aton-detection.patch` - -Build: Force `inet_aton` detection - -For a reason unknown to me, the build system fails to find `inet_aton` -and tries to redefine it with its own implementation in -`flock_compat.c`. - -## `0003-Build-Patch-Serenity-root-directory-into-libtool.patch` +## `0002-Build-Patch-Serenity-root-directory-into-libtool.patch` Build: Patch Serenity root directory into libtool PHP's libtool does not have sysroot support; this is the minimum change to get PHP to build. +## `0003-Remove-include-of-sys-ipc.h.patch` + +Remove include of sys/ipc.h + + +## `0004-Disable-unsupported-prctl-call.patch` + +Disable unsupported prctl call + +Serenity has prctl but doesn't define PR_SET_PDEATHSIG. + diff --git a/Ports/xz/package.sh b/Ports/xz/package.sh index ceb93c79fe0e39..6ea2cb032e83e3 100755 --- a/Ports/xz/package.sh +++ b/Ports/xz/package.sh @@ -1,12 +1,12 @@ #!/usr/bin/env -S bash ../.port_include.sh port='xz' -version='5.6.2' +version='5.8.1' depends=( 'libiconv' 'zlib' ) files=( - "https://tukaani.org/xz/xz-${version}.tar.gz#8bfd20c0e1d86f0402f2497cfa71c6ab62d4cd35fd704276e3140bfb71414519" + "https://tukaani.org/xz/xz-${version}.tar.gz#507825b599356c10dca1cd720c9d0d0c9d5400b9de300af00e4d1ea150795543" ) useconfigure='true' use_fresh_config_sub='true' diff --git a/Ports/xz/patches/0001-libtool-Enable-shared-library-support-for-SerenityOS.patch b/Ports/xz/patches/0001-libtool-Enable-shared-library-support-for-SerenityOS.patch index 778b966ea0c78c..bf64b20d9deb7e 100644 --- a/Ports/xz/patches/0001-libtool-Enable-shared-library-support-for-SerenityOS.patch +++ b/Ports/xz/patches/0001-libtool-Enable-shared-library-support-for-SerenityOS.patch @@ -17,10 +17,10 @@ static library into a shared library. 1 file changed, 23 insertions(+) diff --git a/configure b/configure -index 897a25f77f257182ccd76245b447058199c3478b..aa9ea19199c408d8036c0a2d72d358ea9392b29e 100755 +index 4f89118495be24a8a5e228ad6ff00a6da9d58f9d..8849cd1ad50ec1015b0bcbd7bdce9d18e74e111f 100755 --- a/configure +++ b/configure -@@ -9810,6 +9810,10 @@ tpf*) +@@ -10032,6 +10032,10 @@ tpf*) os2*) lt_cv_deplibs_check_method=pass_all ;; @@ -31,7 +31,7 @@ index 897a25f77f257182ccd76245b447058199c3478b..aa9ea19199c408d8036c0a2d72d358ea esac ;; esac -@@ -13338,6 +13342,10 @@ lt_prog_compiler_static= +@@ -13663,6 +13667,10 @@ lt_prog_compiler_static= lt_prog_compiler_static='-Bstatic' ;; @@ -42,7 +42,7 @@ index 897a25f77f257182ccd76245b447058199c3478b..aa9ea19199c408d8036c0a2d72d358ea *) lt_prog_compiler_can_build_shared=no ;; -@@ -14880,6 +14888,10 @@ printf "%s\n" "$lt_cv_irix_exported_symbol" >&6; } +@@ -15216,6 +15224,10 @@ printf "%s\n" "$lt_cv_irix_exported_symbol" >&6; } hardcode_shlibpath_var=no ;; @@ -53,7 +53,7 @@ index 897a25f77f257182ccd76245b447058199c3478b..aa9ea19199c408d8036c0a2d72d358ea *) ld_shlibs=no ;; -@@ -15955,6 +15967,17 @@ uts4*) +@@ -16362,6 +16374,17 @@ uts4*) shlibpath_var=LD_LIBRARY_PATH ;; @@ -68,6 +68,6 @@ index 897a25f77f257182ccd76245b447058199c3478b..aa9ea19199c408d8036c0a2d72d358ea + dynamic_linker='SerenityOS LibELF' + ;; + - *) - dynamic_linker=no - ;; + emscripten*) + version_type=none + need_lib_prefix=no diff --git a/Ports/xz/patches/0002-liblzma-Don-t-assume-getauxval-is-Linux-only.patch b/Ports/xz/patches/0002-liblzma-Don-t-assume-getauxval-is-Linux-only.patch index 460a9941e781b7..453d53d4b2cb5d 100644 --- a/Ports/xz/patches/0002-liblzma-Don-t-assume-getauxval-is-Linux-only.patch +++ b/Ports/xz/patches/0002-liblzma-Don-t-assume-getauxval-is-Linux-only.patch @@ -11,10 +11,10 @@ in the auxiliary vector. 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/liblzma/check/crc32_arm64.h b/src/liblzma/check/crc32_arm64.h -index 39c1c63ec0eced25ad5042e1611ecd8064831d8d..bea249c9462c7298bce154a1228dba71c4f0e128 100644 +index fb0e8f0105a9e7ddf5abb3a8f8ef3bb2b48ab128..32072922d6b8e3cee8705d0169a1d083c8e08ad6 100644 --- a/src/liblzma/check/crc32_arm64.h +++ b/src/liblzma/check/crc32_arm64.h -@@ -78,7 +78,7 @@ crc32_arch_optimized(const uint8_t *buf, size_t size, uint32_t crc) +@@ -103,7 +103,7 @@ crc32_arch_optimized(const uint8_t *buf, size_t size, uint32_t crc) static inline bool is_arch_extension_supported(void) { @@ -24,15 +24,15 @@ index 39c1c63ec0eced25ad5042e1611ecd8064831d8d..bea249c9462c7298bce154a1228dba71 #elif defined(HAVE_ELF_AUX_INFO) diff --git a/src/liblzma/check/crc_common.h b/src/liblzma/check/crc_common.h -index 63a7b5cefebffc506dc54a6378134008ead3c806..917bf98c9ce611fc946f8acad0c1cf31f2a2fdee 100644 +index 7ea1e60b043bf86dfe421e7594cb01812c51caa0..30a7b5c1a8f4185d76c926c3b41a5488b72ce6ed 100644 --- a/src/liblzma/check/crc_common.h +++ b/src/liblzma/check/crc_common.h -@@ -48,7 +48,7 @@ - #endif - +@@ -89,7 +89,7 @@ extern const uint64_t lzma_crc64_table[4][256]; + // ARM64 + // // Keep this in sync with changes to crc32_arm64.h -#if defined(_WIN32) || defined(HAVE_GETAUXVAL) \ +#if defined(_WIN32) || (defined(__linux__) && defined(HAVE_GETAUXVAL)) \ || defined(HAVE_ELF_AUX_INFO) \ || (defined(__APPLE__) && defined(HAVE_SYSCTLBYNAME)) - # define ARM64_RUNTIME_DETECTION 1 + # define CRC_ARM64_RUNTIME_DETECTION 1 diff --git a/Tests/AK/CMakeLists.txt b/Tests/AK/CMakeLists.txt index e8bb8a1c3a0d4d..d9da0cc3b47449 100644 --- a/Tests/AK/CMakeLists.txt +++ b/Tests/AK/CMakeLists.txt @@ -51,6 +51,7 @@ set(AK_TEST_SOURCES TestIndexSequence.cpp TestInsertionSort.cpp TestIntegerMath.cpp + TestInternetChecksum.cpp TestIntrusiveList.cpp TestIntrusiveRedBlackTree.cpp TestJSON.cpp diff --git a/Tests/AK/TestEnumerate.cpp b/Tests/AK/TestEnumerate.cpp index dffc2869b62d9e..728962aeab17e5 100644 --- a/Tests/AK/TestEnumerate.cpp +++ b/Tests/AK/TestEnumerate.cpp @@ -49,3 +49,43 @@ TEST_CASE(enumerate) EXPECT_EQ(result, (Vector { { 0, 9 }, { 1, 8 }, { 2, 7 }, { 3, 6 } })); } } + +class CopyCounter { +public: + static inline size_t copy_count = 0; + + CopyCounter() = default; + CopyCounter(CopyCounter const&) { ++copy_count; } + CopyCounter(CopyCounter&&) { } + + auto begin() const { return m_vec.begin(); } + auto end() const { return m_vec.end(); } + +private: + Vector m_vec { 1, 2, 3, 4 }; +}; + +TEST_CASE(do_not_copy) +{ + { + Vector result; + CopyCounter::copy_count = 0; + CopyCounter counter {}; + + for (auto [i, value] : enumerate(counter)) + result.append({ i, value }); + + EXPECT_EQ(result, (Vector { { 0, 1 }, { 1, 2 }, { 2, 3 }, { 3, 4 } })); + EXPECT_EQ(CopyCounter::copy_count, 0uz); + } + { + Vector result; + CopyCounter::copy_count = 0; + + for (auto [i, value] : enumerate(CopyCounter {})) + result.append({ i, value }); + + EXPECT_EQ(result, (Vector { { 0, 1 }, { 1, 2 }, { 2, 3 }, { 3, 4 } })); + EXPECT_EQ(CopyCounter::copy_count, 0uz); + } +} diff --git a/Tests/AK/TestInternetChecksum.cpp b/Tests/AK/TestInternetChecksum.cpp new file mode 100644 index 00000000000000..a6cb1374e2055b --- /dev/null +++ b/Tests/AK/TestInternetChecksum.cpp @@ -0,0 +1,28 @@ +/* + * Copyright (c) 2025, the SerenityOS developers. + * + * SPDX-License-Identifier: BSD-2-Clause + */ + +#include +#include +#include +#include + +TEST_CASE(test_internetchecksum) +{ + auto do_test = [](ReadonlyBytes input, u16 expected_result) { + auto digest = InternetChecksum(input).digest(); + EXPECT_EQ(digest, expected_result); + }; + + do_test(to_readonly_bytes(Array { htons(0b0110'0110'0110'0000), htons(0b0101'0101'0101'0101), htons(0b1000'1111'0000'1100) }.span()), 0b1011'0101'0011'1101); + + // Test case from RFC1071. + // The specified result doesn't include the final conversion from one's complement, + // hence the bitwise negation. + do_test(to_readonly_bytes(Array { 0x0100, 0x03f2, 0xf5f4, 0xf7f6 }.span()), static_cast(~0xddf2)); + + // Variation of the above (with an uneven payload). + do_test(to_readonly_bytes(Array { 0x01, 0x00, 0x03 }.span()), htons(static_cast(~0x4))); +} diff --git a/Tests/CMakeLists.txt b/Tests/CMakeLists.txt index 606d7e37cb0b7b..95a00c25984470 100644 --- a/Tests/CMakeLists.txt +++ b/Tests/CMakeLists.txt @@ -5,36 +5,37 @@ add_subdirectory(LibC) add_subdirectory(LibCompress) add_subdirectory(LibCore) add_subdirectory(LibCpp) +add_subdirectory(LibCrypto) add_subdirectory(LibDeviceTree) add_subdirectory(LibDiff) add_subdirectory(LibDisassembly) add_subdirectory(LibEDID) add_subdirectory(LibELF) -add_subdirectory(LibGfx) add_subdirectory(LibGL) add_subdirectory(LibGLSL) +add_subdirectory(LibGfx) add_subdirectory(LibHID) add_subdirectory(LibIMAP) add_subdirectory(LibJS) +add_subdirectory(LibLine) add_subdirectory(LibLocale) add_subdirectory(LibMarkdown) +add_subdirectory(LibMedia) add_subdirectory(LibPDF) add_subdirectory(LibRegex) +add_subdirectory(LibSQL) add_subdirectory(LibSemVer) add_subdirectory(LibShell) -add_subdirectory(LibSQL) +add_subdirectory(LibTLS) add_subdirectory(LibTest) add_subdirectory(LibTextCodec) add_subdirectory(LibThreading) add_subdirectory(LibTimeZone) -add_subdirectory(LibUnicode) add_subdirectory(LibURL) -add_subdirectory(LibMedia) +add_subdirectory(LibUnicode) add_subdirectory(LibWasm) add_subdirectory(LibWeb) add_subdirectory(LibWebView) add_subdirectory(LibXML) -add_subdirectory(LibCrypto) -add_subdirectory(LibTLS) add_subdirectory(Spreadsheet) add_subdirectory(Utilities) diff --git a/Tests/Kernel/CMakeLists.txt b/Tests/Kernel/CMakeLists.txt index a947ff6aa00229..796128c87be453 100644 --- a/Tests/Kernel/CMakeLists.txt +++ b/Tests/Kernel/CMakeLists.txt @@ -38,7 +38,6 @@ target_link_libraries(fuzz-syscalls PRIVATE LibSystem) serenity_test("crash.cpp" Kernel MAIN_ALREADY_DEFINED) set(LIBTEST_BASED_SOURCES - TestAnonymousMmap.cpp TestEFault.cpp TestEmptyPrivateInodeVMObject.cpp TestEmptySharedInodeVMObject.cpp @@ -60,7 +59,7 @@ set(LIBTEST_BASED_SOURCES TestProcFSWrite.cpp TestSigAltStack.cpp TestSigHandler.cpp - TestSigWait.cpp + TestSignalDispatch.cpp TestTCPSocket.cpp TestWait.cpp TestWXProtection.cpp @@ -70,6 +69,14 @@ if (ENABLE_KERNEL_COVERAGE_COLLECTION) list(APPEND LIBTEST_BASED_SOURCES TestKCOV.cpp) endif() +if (NOT ENABLE_USERSPACE_COVERAGE_COLLECTION) + # FIXME: Make these tests generate valid coverage files. + list(APPEND LIBTEST_BASED_SOURCES + TestAnonymousMmap.cpp + TestSigWait.cpp + ) +endif() + if (CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64") list(APPEND LIBTEST_BASED_SOURCES TestMemoryDeviceMmap.cpp) endif() diff --git a/Tests/Kernel/TestProcFSWrite.cpp b/Tests/Kernel/TestProcFSWrite.cpp index 0245e99138f5c3..64085231276f71 100644 --- a/Tests/Kernel/TestProcFSWrite.cpp +++ b/Tests/Kernel/TestProcFSWrite.cpp @@ -41,3 +41,16 @@ TEST_CASE(root_writes_to_procfs) FAIL("Wrote successfully?!"); } } + +TEST_CASE(set_coredump_path) +{ + auto fd = open("/sys/kernel/conf/coredump_directory", O_RDWR); + if (fd < 0) { + perror("open"); + FAIL("open failed?! See debugout"); + return; + } + static constexpr auto path = "relative/path"sv; + write(fd, path.characters_without_null_termination(), path.length()); + EXPECT_EQ(errno, EINVAL); +} diff --git a/Tests/Kernel/TestSignalDispatch.cpp b/Tests/Kernel/TestSignalDispatch.cpp new file mode 100644 index 00000000000000..936d6ffcda63a4 --- /dev/null +++ b/Tests/Kernel/TestSignalDispatch.cpp @@ -0,0 +1,62 @@ +/* + * Copyright (c) 2025, the SerenityOS developers. + * + * SPDX-License-Identifier: BSD-2-Clause + */ + +#include +#include +#include +#include +#include + +// These will eventually point to shared mmaps which will be used as a crude form of IPC. +static bool volatile* s_received_sigint = nullptr; +static bool volatile* s_ready_to_receive_signal = nullptr; + +static void handle_sigint(int) +{ + *s_received_sigint = true; +} + +TEST_CASE(signal_dispatch_to_spinning_thread) +{ + s_received_sigint = reinterpret_cast(mmap(nullptr, sizeof(bool), PROT_READ | PROT_WRITE, MAP_SHARED | MAP_ANONYMOUS, -1, 0)); + s_ready_to_receive_signal = reinterpret_cast(mmap(nullptr, sizeof(bool), PROT_READ | PROT_WRITE, MAP_SHARED | MAP_ANONYMOUS, -1, 0)); + + VERIFY(s_received_sigint); + VERIFY(s_ready_to_receive_signal); + + pid_t pid = fork(); + VERIFY(pid != -1); + + if (pid == 0) { + VERIFY(signal(SIGINT, handle_sigint) != SIG_ERR); + *s_ready_to_receive_signal = true; + while (1) + continue; + VERIFY_NOT_REACHED(); + } + + auto start_time = MonotonicTime::now(); + while (*s_ready_to_receive_signal == false) { + if ((MonotonicTime::now() - start_time).to_truncated_seconds() >= 5) { + FAIL("Timed out while waiting for signal handler creation"); + return; + } + } + + VERIFY(kill(pid, SIGINT) == 0); + + start_time = MonotonicTime::now(); + while (*s_received_sigint == false) { + if ((MonotonicTime::now() - start_time).to_truncated_seconds() >= 5) { + FAIL("Timed out while waiting for SIGINT"); + return; + } + } + + // Normally this should be done in a ScopeGuard, but that's kind of moot here + // because this signal likely won't be handled properly anyway if this test failed. + VERIFY(kill(pid, SIGKILL) == 0); +} diff --git a/Tests/LibCrypto/TestChecksum.cpp b/Tests/LibCrypto/TestChecksum.cpp index 7b8a2c5eface65..5d9454461aff1d 100644 --- a/Tests/LibCrypto/TestChecksum.cpp +++ b/Tests/LibCrypto/TestChecksum.cpp @@ -6,7 +6,6 @@ #include #include -#include #include #include #include @@ -65,13 +64,3 @@ TEST_CASE(test_crc32) do_test("The quick brown fox jumps over the lazy dog"sv.bytes(), 0x414FA339); do_test("various CRC algorithms input data"sv.bytes(), 0x9BD366AE); } - -TEST_CASE(test_ipv4header) -{ - auto do_test = [](ReadonlyBytes input, u16 expected_result) { - auto digest = Crypto::Checksum::IPv4Header(input).digest(); - EXPECT_EQ(digest, expected_result); - }; - Array bytes = { htons(0b0110'0110'0110'0000), htons(0b0101'0101'0101'0101), htons(0b1000'1111'0000'1100) }; - do_test(to_readonly_bytes(bytes.span()), htons(0b1011'0101'0011'1101)); -} diff --git a/Tests/LibGfx/TestBilevelImage.cpp b/Tests/LibGfx/TestBilevelImage.cpp index f7dce5fa0f5f23..69c82b90b1a744 100644 --- a/Tests/LibGfx/TestBilevelImage.cpp +++ b/Tests/LibGfx/TestBilevelImage.cpp @@ -4,6 +4,9 @@ * SPDX-License-Identifier: BSD-2-Clause */ +#include +#include +#include #include #include @@ -60,3 +63,45 @@ TEST_CASE(get_bits_over_8bits) EXPECT_EQ(bilevel->get_bits(8, 0, 8), 0xFE); EXPECT_EQ(bilevel->get_bits(12, 0, 4), 0xE); } + +static ErrorOr test_bayer_dither(Gfx::DitheringAlgorithm algorithm, u32 size) +{ + auto srgb_curve = TRY(Gfx::ICC::sRGB_curve()); + auto bitmap = TRY(Gfx::Bitmap::create(Gfx::BitmapFormat::BGRA8888, { size, size })); + auto number_of_states = size * size + 1; + + auto test_luminosity = [&](f32 input_luminosity, f32 output_luminosity) -> ErrorOr { + auto uncompressed = round_to(srgb_curve->evaluate_inverse(input_luminosity) * 255); + bitmap->fill(Gfx::Color(uncompressed, uncompressed, uncompressed)); + auto bilevel = TRY(Gfx::BilevelImage::create_from_bitmap(bitmap, algorithm)); + double average = 0; + for (u32 y = 0; y < bilevel->height(); ++y) { + for (u32 x = 0; x < bilevel->width(); ++x) + average += bilevel->get_bit(x, y) ? 0 : 1; + } + + EXPECT_APPROXIMATE(average / (size * size), output_luminosity); + return {}; + }; + + // Test full black and full white. + TRY(test_luminosity(0, 0)); + TRY(test_luminosity(1, 1)); + + // Test all states at half the range. + for (u32 s = 0; s < number_of_states; ++s) { + // We test all states in the middle of there range. + auto value = (static_cast(s) + 0.5f) / number_of_states; + // There are only (number_of_states - 1) states of luminosity. + TRY(test_luminosity(value, static_cast(s) / (number_of_states - 1))); + } + return {}; +} + +TEST_CASE(bayer_dither) +{ + + TRY_OR_FAIL(test_bayer_dither(Gfx::DitheringAlgorithm::Bayer2x2, 2)); + TRY_OR_FAIL(test_bayer_dither(Gfx::DitheringAlgorithm::Bayer4x4, 4)); + TRY_OR_FAIL(test_bayer_dither(Gfx::DitheringAlgorithm::Bayer8x8, 8)); +} diff --git a/Tests/LibGfx/TestICCProfile.cpp b/Tests/LibGfx/TestICCProfile.cpp index 55df31f48612e4..00b3c93e86f93e 100644 --- a/Tests/LibGfx/TestICCProfile.cpp +++ b/Tests/LibGfx/TestICCProfile.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2023, Nico Weber + * Copyright (c) 2023-2025, Nico Weber * * SPDX-License-Identifier: BSD-2-Clause */ @@ -116,7 +116,7 @@ TEST_CASE(built_in_sRGB) EXPECT(memmem(serialized_bytes.data(), serialized_bytes.size(), sf32, sizeof(sf32)) != nullptr); } -TEST_CASE(to_pcs) +TEST_CASE(sRGB_to_pcs) { auto sRGB = MUST(Gfx::ICC::sRGB()); EXPECT(sRGB->data_color_space() == Gfx::ICC::ColorSpace::RGB); @@ -174,7 +174,7 @@ TEST_CASE(to_pcs) EXPECT_APPROXIMATE_VECTOR3(xyz_from_sRGB(64, 128, 192), r_xyz * f64 + g_xyz * f128 + b_xyz * f192); } -TEST_CASE(from_pcs) +TEST_CASE(sRBB_from_pcs) { auto sRGB = MUST(Gfx::ICC::sRGB()); @@ -224,7 +224,7 @@ TEST_CASE(from_pcs) EXPECT_EQ(sRGB_from_xyz(r_xyz * f64 + g_xyz * f128 + b_xyz * f192), Color(64, 128, 192)); } -TEST_CASE(to_lab) +TEST_CASE(sRGB_to_lab) { auto sRGB = MUST(Gfx::ICC::sRGB()); auto lab_from_sRGB = [&sRGB](u8 r, u8 g, u8 b) { @@ -274,6 +274,80 @@ TEST_CASE(to_lab) EXPECT_APPROXIMATE_LAB(lab_from_sRGB(255, 255, 255), expected[7]); } +static void test_roundtrip(Gfx::ICC::Profile const& profile) +{ + // Ideally this would be 1, but that makes tests take a few minutes on fast machine. + // It's supposed to pass with 1, though. + int const increment = 7; + for (int a = 0; a < 256; a += increment) { + for (int b = 0; b < 256; b += increment) { + for (int c = 0; c < 256; c += increment) { + u8 color_in[3] = { static_cast(a), static_cast(b), static_cast(c) }; + u8 color_out[3]; + auto pcs = MUST(profile.to_pcs(color_in)); + MUST(profile.from_pcs(profile, pcs, color_out)); + EXPECT_EQ(color_in[0], color_out[0]); + EXPECT_EQ(color_in[1], color_out[1]); + EXPECT_EQ(color_in[2], color_out[2]); + } + } + } +} + +TEST_CASE(roundtrip_lab_mft1) +{ + auto profile = TRY_OR_FAIL(Gfx::ICC::IdentityLAB()); + test_roundtrip(*profile); +} + +TEST_CASE(roundtrip_lab_mft2) +{ + auto profile = TRY_OR_FAIL(Gfx::ICC::IdentityLAB_mft2()); + test_roundtrip(*profile); +} + +TEST_CASE(roundtrip_lab_mABmBA_no_clut) +{ + auto profile = TRY_OR_FAIL(Gfx::ICC::IdentityLAB_mABmBA_no_clut()); + test_roundtrip(*profile); +} + +TEST_CASE(roundtrip_lab_mABmBA_u8_clut) +{ + auto profile = TRY_OR_FAIL(Gfx::ICC::IdentityLAB_mABmBA_u8_clut()); + test_roundtrip(*profile); +} + +TEST_CASE(roundtrip_lab_mABmBA_u16_clut) +{ + auto profile = TRY_OR_FAIL(Gfx::ICC::IdentityLAB_mABmBA_u16_clut()); + test_roundtrip(*profile); +} + +TEST_CASE(roundtrip_xyz_mft2) +{ + auto profile = TRY_OR_FAIL(Gfx::ICC::IdentityXYZ_D50()); + test_roundtrip(*profile); +} + +TEST_CASE(roundtrip_sRGB_matrix_profile) +{ + auto profile = TRY_OR_FAIL(Gfx::ICC::sRGB()); + test_roundtrip(*profile); +} + +TEST_CASE(roundtrip_xyz_mABmBA_no_clut) +{ + auto profile = TRY_OR_FAIL(Gfx::ICC::IdentityXYZ_D50_mABmBA_no_clut()); + test_roundtrip(*profile); +} + +TEST_CASE(roundtrip_xyz_mABmBA_u16_clut) +{ + auto profile = TRY_OR_FAIL(Gfx::ICC::IdentityXYZ_D50_mABmBA_u16_clut()); + test_roundtrip(*profile); +} + TEST_CASE(malformed_profile) { Array test_inputs = { diff --git a/Tests/LibGfx/TestImageDecoder.cpp b/Tests/LibGfx/TestImageDecoder.cpp index 8f422aea8c2b3b..f2995627ea23a1 100644 --- a/Tests/LibGfx/TestImageDecoder.cpp +++ b/Tests/LibGfx/TestImageDecoder.cpp @@ -334,12 +334,17 @@ TEST_CASE(test_jbig2_decode) Array test_inputs = { TEST_INPUT("jbig2/bitmap.jbig2"sv), + TEST_INPUT("jbig2/bitmap-randomaccess.jbig2"sv), TEST_INPUT("jbig2/bitmap-p32-eof.jbig2"sv), TEST_INPUT("jbig2/bitmap-initially-unknown-size.jbig2"sv), TEST_INPUT("jbig2/bitmap-composite-and-xnor.jbig2"sv), + TEST_INPUT("jbig2/bitmap-composite-and-xnor-halftone.jbig2"sv), TEST_INPUT("jbig2/bitmap-composite-and-xnor-refine.jbig2"sv), + TEST_INPUT("jbig2/bitmap-composite-and-xnor-text.jbig2"sv), TEST_INPUT("jbig2/bitmap-composite-or-xor-replace.jbig2"sv), + TEST_INPUT("jbig2/bitmap-composite-or-xor-replace-halftone.jbig2"sv), TEST_INPUT("jbig2/bitmap-composite-or-xor-replace-refine.jbig2"sv), + TEST_INPUT("jbig2/bitmap-composite-or-xor-replace-text.jbig2"sv), TEST_INPUT("jbig2/bitmap-customat.jbig2"sv), TEST_INPUT("jbig2/bitmap-tpgdon.jbig2"sv), TEST_INPUT("jbig2/bitmap-customat-tpgdon.jbig2"sv), @@ -365,6 +370,8 @@ TEST_CASE(test_jbig2_decode) TEST_INPUT("jbig2/bitmap-trailing-7fff-stripped-harder.jbig2"sv), TEST_INPUT("jbig2/bitmap-trailing-7fff-stripped-harder-refine.jbig2"sv), TEST_INPUT("jbig2/bitmap-refine.jbig2"sv), + TEST_INPUT("jbig2/bitmap-refine-page.jbig2"sv), + TEST_INPUT("jbig2/bitmap-refine-page-subrect.jbig2"sv), TEST_INPUT("jbig2/bitmap-refine-customat.jbig2"sv), TEST_INPUT("jbig2/bitmap-refine-lossless.jbig2"sv), TEST_INPUT("jbig2/bitmap-refine-refine.jbig2"sv), @@ -372,19 +379,32 @@ TEST_CASE(test_jbig2_decode) TEST_INPUT("jbig2/bitmap-refine-template1.jbig2"sv), TEST_INPUT("jbig2/bitmap-refine-template1-tpgron.jbig2"sv), TEST_INPUT("jbig2/bitmap-halftone.jbig2"sv), + TEST_INPUT("jbig2/bitmap-halftone-composite.jbig2"sv), + TEST_INPUT("jbig2/bitmap-halftone-grid.jbig2"sv), + TEST_INPUT("jbig2/bitmap-halftone-refine.jbig2"sv), + TEST_INPUT("jbig2/bitmap-halftone-skip-dummy.jbig2"sv), + TEST_INPUT("jbig2/bitmap-halftone-skip-grid.jbig2"sv), + TEST_INPUT("jbig2/bitmap-halftone-skip-grid-template1.jbig2"sv), + TEST_INPUT("jbig2/bitmap-halftone-skip-grid-template2.jbig2"sv), + TEST_INPUT("jbig2/bitmap-halftone-skip-grid-template3.jbig2"sv), TEST_INPUT("jbig2/bitmap-halftone-template1.jbig2"sv), TEST_INPUT("jbig2/bitmap-halftone-template2.jbig2"sv), TEST_INPUT("jbig2/bitmap-halftone-template3.jbig2"sv), TEST_INPUT("jbig2/bitmap-halftone-10bpp.jbig2"sv), TEST_INPUT("jbig2/bitmap-halftone-10bpp-mmr.jbig2"sv), TEST_INPUT("jbig2/bitmap-symbol.jbig2"sv), + TEST_INPUT("jbig2/bitmap-symbol-big-segmentid.jbig2"sv), + TEST_INPUT("jbig2/bitmap-symbol-context-reuse.jbig2"sv), + TEST_INPUT("jbig2/bitmap-symbol-empty.jbig2"sv), TEST_INPUT("jbig2/bitmap-symbol-manyrefs.jbig2"sv), TEST_INPUT("jbig2/bitmap-symbol-negative-sbdsoffset.jbig2"sv), + TEST_INPUT("jbig2/bitmap-symbol-refine.jbig2"sv), TEST_INPUT("jbig2/bitmap-symbol-symhuff-texthuff.jbig2"sv), TEST_INPUT("jbig2/bitmap-symbol-symhuffB5B3-texthuffB7B9B12.jbig2"sv), TEST_INPUT("jbig2/bitmap-symbol-symhuff-texthuffB10B13.jbig2"sv), TEST_INPUT("jbig2/bitmap-symbol-symhuffcustom-texthuffcustom.jbig2"sv), TEST_INPUT("jbig2/bitmap-symbol-symhuffuncompressed-texthuff.jbig2"sv), + TEST_INPUT("jbig2/bitmap-symbol-textcomposite.jbig2"sv), TEST_INPUT("jbig2/bitmap-symbol-textrefine.jbig2"sv), TEST_INPUT("jbig2/bitmap-symbol-textrefine-customat.jbig2"sv), TEST_INPUT("jbig2/bitmap-symbol-textrefine-negative-delta-width.jbig2"sv), @@ -401,6 +421,8 @@ TEST_CASE(test_jbig2_decode) TEST_INPUT("jbig2/bitmap-symbol-symbolrefineseveral.jbig2"sv), TEST_INPUT("jbig2/bitmap-symbol-symhuffrefineone.jbig2"sv), TEST_INPUT("jbig2/bitmap-symbol-symhuffrefineseveral.jbig2"sv), + TEST_INPUT("jbig2/bitmap-symbol-symbolrefine-textrefine.jbig2"sv), + TEST_INPUT("jbig2/bitmap-symbol-symhuffrefine-textrefine.jbig2"sv), TEST_INPUT("jbig2/bitmap-symbol-textbottomleft.jbig2"sv), TEST_INPUT("jbig2/bitmap-symbol-textbottomlefttranspose.jbig2"sv), TEST_INPUT("jbig2/bitmap-symbol-textbottomright.jbig2"sv), @@ -408,18 +430,7 @@ TEST_CASE(test_jbig2_decode) TEST_INPUT("jbig2/bitmap-symbol-texttopright.jbig2"sv), TEST_INPUT("jbig2/bitmap-symbol-texttoprighttranspose.jbig2"sv), TEST_INPUT("jbig2/bitmap-symbol-texttranspose.jbig2"sv), - // Missing tests, in part because there's no easy way to generate them: - // - lossless halftone (code support added in #26043) - // - rotated halftone (code support added in #26044) - // - composition test for halftone regions - // - intermediate halftone / refinement of halftone regions - // - negative position in composition (code support added in #26046) - // - intermediate text regions (code support added in #26197) - // - intermediate halftone regions (code support added in #26197) - // - intermediate direct regions (code support added in #26197) - // - symbol refinement referring to symbol in same segment // Missing tests for things that aren't implemented yet: - // - immediate refinement regions not referring to a direct region (i.e. refining the page) // - exttemplate // - colors }; diff --git a/Tests/LibGfx/test-inputs/jbig2/bitmap-composite-and-xnor-halftone.jbig2 b/Tests/LibGfx/test-inputs/jbig2/bitmap-composite-and-xnor-halftone.jbig2 new file mode 100644 index 00000000000000..bec49964529a5b Binary files /dev/null and b/Tests/LibGfx/test-inputs/jbig2/bitmap-composite-and-xnor-halftone.jbig2 differ diff --git a/Tests/LibGfx/test-inputs/jbig2/bitmap-composite-and-xnor-text.jbig2 b/Tests/LibGfx/test-inputs/jbig2/bitmap-composite-and-xnor-text.jbig2 new file mode 100644 index 00000000000000..a007126464b6f6 Binary files /dev/null and b/Tests/LibGfx/test-inputs/jbig2/bitmap-composite-and-xnor-text.jbig2 differ diff --git a/Tests/LibGfx/test-inputs/jbig2/bitmap-composite-or-xor-replace-halftone.jbig2 b/Tests/LibGfx/test-inputs/jbig2/bitmap-composite-or-xor-replace-halftone.jbig2 new file mode 100644 index 00000000000000..32ce47be34f3c9 Binary files /dev/null and b/Tests/LibGfx/test-inputs/jbig2/bitmap-composite-or-xor-replace-halftone.jbig2 differ diff --git a/Tests/LibGfx/test-inputs/jbig2/bitmap-composite-or-xor-replace-refine.jbig2 b/Tests/LibGfx/test-inputs/jbig2/bitmap-composite-or-xor-replace-refine.jbig2 index 3e26175a045d36..24c318e5c0d1f0 100644 Binary files a/Tests/LibGfx/test-inputs/jbig2/bitmap-composite-or-xor-replace-refine.jbig2 and b/Tests/LibGfx/test-inputs/jbig2/bitmap-composite-or-xor-replace-refine.jbig2 differ diff --git a/Tests/LibGfx/test-inputs/jbig2/bitmap-composite-or-xor-replace-text.jbig2 b/Tests/LibGfx/test-inputs/jbig2/bitmap-composite-or-xor-replace-text.jbig2 new file mode 100644 index 00000000000000..4f47e9483fdc18 Binary files /dev/null and b/Tests/LibGfx/test-inputs/jbig2/bitmap-composite-or-xor-replace-text.jbig2 differ diff --git a/Tests/LibGfx/test-inputs/jbig2/bitmap-composite-or-xor-replace.jbig2 b/Tests/LibGfx/test-inputs/jbig2/bitmap-composite-or-xor-replace.jbig2 index 475488145e24b6..6e568fc21713d3 100644 Binary files a/Tests/LibGfx/test-inputs/jbig2/bitmap-composite-or-xor-replace.jbig2 and b/Tests/LibGfx/test-inputs/jbig2/bitmap-composite-or-xor-replace.jbig2 differ diff --git a/Tests/LibGfx/test-inputs/jbig2/bitmap-halftone-composite.jbig2 b/Tests/LibGfx/test-inputs/jbig2/bitmap-halftone-composite.jbig2 new file mode 100644 index 00000000000000..11f287af6678e3 Binary files /dev/null and b/Tests/LibGfx/test-inputs/jbig2/bitmap-halftone-composite.jbig2 differ diff --git a/Tests/LibGfx/test-inputs/jbig2/bitmap-halftone-grid.jbig2 b/Tests/LibGfx/test-inputs/jbig2/bitmap-halftone-grid.jbig2 new file mode 100644 index 00000000000000..ae6e8d50f9aefc Binary files /dev/null and b/Tests/LibGfx/test-inputs/jbig2/bitmap-halftone-grid.jbig2 differ diff --git a/Tests/LibGfx/test-inputs/jbig2/bitmap-halftone-refine.jbig2 b/Tests/LibGfx/test-inputs/jbig2/bitmap-halftone-refine.jbig2 new file mode 100644 index 00000000000000..cf5a92356c1d55 Binary files /dev/null and b/Tests/LibGfx/test-inputs/jbig2/bitmap-halftone-refine.jbig2 differ diff --git a/Tests/LibGfx/test-inputs/jbig2/bitmap-halftone-skip-dummy.jbig2 b/Tests/LibGfx/test-inputs/jbig2/bitmap-halftone-skip-dummy.jbig2 new file mode 100644 index 00000000000000..b44196206a7b2b Binary files /dev/null and b/Tests/LibGfx/test-inputs/jbig2/bitmap-halftone-skip-dummy.jbig2 differ diff --git a/Tests/LibGfx/test-inputs/jbig2/bitmap-halftone-skip-grid-template1.jbig2 b/Tests/LibGfx/test-inputs/jbig2/bitmap-halftone-skip-grid-template1.jbig2 new file mode 100644 index 00000000000000..b9479144137323 Binary files /dev/null and b/Tests/LibGfx/test-inputs/jbig2/bitmap-halftone-skip-grid-template1.jbig2 differ diff --git a/Tests/LibGfx/test-inputs/jbig2/bitmap-halftone-skip-grid-template2.jbig2 b/Tests/LibGfx/test-inputs/jbig2/bitmap-halftone-skip-grid-template2.jbig2 new file mode 100644 index 00000000000000..244095268c34ae Binary files /dev/null and b/Tests/LibGfx/test-inputs/jbig2/bitmap-halftone-skip-grid-template2.jbig2 differ diff --git a/Tests/LibGfx/test-inputs/jbig2/bitmap-halftone-skip-grid-template3.jbig2 b/Tests/LibGfx/test-inputs/jbig2/bitmap-halftone-skip-grid-template3.jbig2 new file mode 100644 index 00000000000000..c41de28211ba19 Binary files /dev/null and b/Tests/LibGfx/test-inputs/jbig2/bitmap-halftone-skip-grid-template3.jbig2 differ diff --git a/Tests/LibGfx/test-inputs/jbig2/bitmap-halftone-skip-grid.jbig2 b/Tests/LibGfx/test-inputs/jbig2/bitmap-halftone-skip-grid.jbig2 new file mode 100644 index 00000000000000..5fc111ae627341 Binary files /dev/null and b/Tests/LibGfx/test-inputs/jbig2/bitmap-halftone-skip-grid.jbig2 differ diff --git a/Tests/LibGfx/test-inputs/jbig2/bitmap-randomaccess.jbig2 b/Tests/LibGfx/test-inputs/jbig2/bitmap-randomaccess.jbig2 new file mode 100644 index 00000000000000..59607809a81ead Binary files /dev/null and b/Tests/LibGfx/test-inputs/jbig2/bitmap-randomaccess.jbig2 differ diff --git a/Tests/LibGfx/test-inputs/jbig2/bitmap-refine-page-subrect.jbig2 b/Tests/LibGfx/test-inputs/jbig2/bitmap-refine-page-subrect.jbig2 new file mode 100644 index 00000000000000..04d00fcd662f52 Binary files /dev/null and b/Tests/LibGfx/test-inputs/jbig2/bitmap-refine-page-subrect.jbig2 differ diff --git a/Tests/LibGfx/test-inputs/jbig2/bitmap-refine-page.jbig2 b/Tests/LibGfx/test-inputs/jbig2/bitmap-refine-page.jbig2 new file mode 100644 index 00000000000000..87c9dc1c975c03 Binary files /dev/null and b/Tests/LibGfx/test-inputs/jbig2/bitmap-refine-page.jbig2 differ diff --git a/Tests/LibGfx/test-inputs/jbig2/bitmap-refine-template1-tpgron.jbig2 b/Tests/LibGfx/test-inputs/jbig2/bitmap-refine-template1-tpgron.jbig2 index ecd3ff52d94809..33dcbdaa65a7da 100644 Binary files a/Tests/LibGfx/test-inputs/jbig2/bitmap-refine-template1-tpgron.jbig2 and b/Tests/LibGfx/test-inputs/jbig2/bitmap-refine-template1-tpgron.jbig2 differ diff --git a/Tests/LibGfx/test-inputs/jbig2/bitmap-refine-tpgron.jbig2 b/Tests/LibGfx/test-inputs/jbig2/bitmap-refine-tpgron.jbig2 index 6a78a4b209eb7b..a716f5da18a2c2 100644 Binary files a/Tests/LibGfx/test-inputs/jbig2/bitmap-refine-tpgron.jbig2 and b/Tests/LibGfx/test-inputs/jbig2/bitmap-refine-tpgron.jbig2 differ diff --git a/Tests/LibGfx/test-inputs/jbig2/bitmap-symbol-big-segmentid.jbig2 b/Tests/LibGfx/test-inputs/jbig2/bitmap-symbol-big-segmentid.jbig2 new file mode 100644 index 00000000000000..bdc4e164ffec75 Binary files /dev/null and b/Tests/LibGfx/test-inputs/jbig2/bitmap-symbol-big-segmentid.jbig2 differ diff --git a/Tests/LibGfx/test-inputs/jbig2/bitmap-symbol-context-reuse.jbig2 b/Tests/LibGfx/test-inputs/jbig2/bitmap-symbol-context-reuse.jbig2 new file mode 100644 index 00000000000000..9366487be0b4e1 Binary files /dev/null and b/Tests/LibGfx/test-inputs/jbig2/bitmap-symbol-context-reuse.jbig2 differ diff --git a/Tests/LibGfx/test-inputs/jbig2/bitmap-symbol-empty.jbig2 b/Tests/LibGfx/test-inputs/jbig2/bitmap-symbol-empty.jbig2 new file mode 100644 index 00000000000000..71035ff75e5196 Binary files /dev/null and b/Tests/LibGfx/test-inputs/jbig2/bitmap-symbol-empty.jbig2 differ diff --git a/Tests/LibGfx/test-inputs/jbig2/bitmap-symbol-negative-sbdsoffset.jbig2 b/Tests/LibGfx/test-inputs/jbig2/bitmap-symbol-negative-sbdsoffset.jbig2 index 737c73c201c67d..58cd6f5ab8efd3 100644 Binary files a/Tests/LibGfx/test-inputs/jbig2/bitmap-symbol-negative-sbdsoffset.jbig2 and b/Tests/LibGfx/test-inputs/jbig2/bitmap-symbol-negative-sbdsoffset.jbig2 differ diff --git a/Tests/LibGfx/test-inputs/jbig2/bitmap-symbol-refine.jbig2 b/Tests/LibGfx/test-inputs/jbig2/bitmap-symbol-refine.jbig2 new file mode 100644 index 00000000000000..1bb6608cd15df2 Binary files /dev/null and b/Tests/LibGfx/test-inputs/jbig2/bitmap-symbol-refine.jbig2 differ diff --git a/Tests/LibGfx/test-inputs/jbig2/bitmap-symbol-symbolrefine-textrefine.jbig2 b/Tests/LibGfx/test-inputs/jbig2/bitmap-symbol-symbolrefine-textrefine.jbig2 new file mode 100644 index 00000000000000..ed266926b1d0e4 Binary files /dev/null and b/Tests/LibGfx/test-inputs/jbig2/bitmap-symbol-symbolrefine-textrefine.jbig2 differ diff --git a/Tests/LibGfx/test-inputs/jbig2/bitmap-symbol-symhuffrefine-textrefine.jbig2 b/Tests/LibGfx/test-inputs/jbig2/bitmap-symbol-symhuffrefine-textrefine.jbig2 new file mode 100644 index 00000000000000..6cc6a511e691d7 Binary files /dev/null and b/Tests/LibGfx/test-inputs/jbig2/bitmap-symbol-symhuffrefine-textrefine.jbig2 differ diff --git a/Tests/LibGfx/test-inputs/jbig2/bitmap-symbol-textbottomleft.jbig2 b/Tests/LibGfx/test-inputs/jbig2/bitmap-symbol-textbottomleft.jbig2 index 18ced24cff494b..ddb46f452376cf 100644 Binary files a/Tests/LibGfx/test-inputs/jbig2/bitmap-symbol-textbottomleft.jbig2 and b/Tests/LibGfx/test-inputs/jbig2/bitmap-symbol-textbottomleft.jbig2 differ diff --git a/Tests/LibGfx/test-inputs/jbig2/bitmap-symbol-textbottomlefttranspose.jbig2 b/Tests/LibGfx/test-inputs/jbig2/bitmap-symbol-textbottomlefttranspose.jbig2 index 3e89c14e26b1ed..5b96f671a3a93e 100644 Binary files a/Tests/LibGfx/test-inputs/jbig2/bitmap-symbol-textbottomlefttranspose.jbig2 and b/Tests/LibGfx/test-inputs/jbig2/bitmap-symbol-textbottomlefttranspose.jbig2 differ diff --git a/Tests/LibGfx/test-inputs/jbig2/bitmap-symbol-textbottomright.jbig2 b/Tests/LibGfx/test-inputs/jbig2/bitmap-symbol-textbottomright.jbig2 index 1c8f2aceaee6c6..bec001c640a574 100644 Binary files a/Tests/LibGfx/test-inputs/jbig2/bitmap-symbol-textbottomright.jbig2 and b/Tests/LibGfx/test-inputs/jbig2/bitmap-symbol-textbottomright.jbig2 differ diff --git a/Tests/LibGfx/test-inputs/jbig2/bitmap-symbol-textbottomrighttranspose.jbig2 b/Tests/LibGfx/test-inputs/jbig2/bitmap-symbol-textbottomrighttranspose.jbig2 index 2dbc96d52838e1..98d5065f5d7b17 100644 Binary files a/Tests/LibGfx/test-inputs/jbig2/bitmap-symbol-textbottomrighttranspose.jbig2 and b/Tests/LibGfx/test-inputs/jbig2/bitmap-symbol-textbottomrighttranspose.jbig2 differ diff --git a/Tests/LibGfx/test-inputs/jbig2/bitmap-symbol-textcomposite.jbig2 b/Tests/LibGfx/test-inputs/jbig2/bitmap-symbol-textcomposite.jbig2 new file mode 100644 index 00000000000000..302a71ecb555b7 Binary files /dev/null and b/Tests/LibGfx/test-inputs/jbig2/bitmap-symbol-textcomposite.jbig2 differ diff --git a/Tests/LibGfx/test-inputs/jbig2/bitmap-symbol-texttopright.jbig2 b/Tests/LibGfx/test-inputs/jbig2/bitmap-symbol-texttopright.jbig2 index 337a34121849ac..89a91d6dac5dea 100644 Binary files a/Tests/LibGfx/test-inputs/jbig2/bitmap-symbol-texttopright.jbig2 and b/Tests/LibGfx/test-inputs/jbig2/bitmap-symbol-texttopright.jbig2 differ diff --git a/Tests/LibGfx/test-inputs/jbig2/bitmap-symbol-texttoprighttranspose.jbig2 b/Tests/LibGfx/test-inputs/jbig2/bitmap-symbol-texttoprighttranspose.jbig2 index 41452982b106ad..56d6f8125d1fb7 100644 Binary files a/Tests/LibGfx/test-inputs/jbig2/bitmap-symbol-texttoprighttranspose.jbig2 and b/Tests/LibGfx/test-inputs/jbig2/bitmap-symbol-texttoprighttranspose.jbig2 differ diff --git a/Tests/LibGfx/test-inputs/jbig2/bitmap-symbol-texttranspose.jbig2 b/Tests/LibGfx/test-inputs/jbig2/bitmap-symbol-texttranspose.jbig2 index 8c730d16ec824e..c4d051fd162894 100644 Binary files a/Tests/LibGfx/test-inputs/jbig2/bitmap-symbol-texttranspose.jbig2 and b/Tests/LibGfx/test-inputs/jbig2/bitmap-symbol-texttranspose.jbig2 differ diff --git a/Tests/LibGfx/test-inputs/jbig2/bitmap-symbol.jbig2 b/Tests/LibGfx/test-inputs/jbig2/bitmap-symbol.jbig2 index 74f9b04e97c38b..9cc96de03e5302 100644 Binary files a/Tests/LibGfx/test-inputs/jbig2/bitmap-symbol.jbig2 and b/Tests/LibGfx/test-inputs/jbig2/bitmap-symbol.jbig2 differ diff --git a/Tests/LibGfx/test-inputs/jbig2/bitmap-template1-tpgdon.jbig2 b/Tests/LibGfx/test-inputs/jbig2/bitmap-template1-tpgdon.jbig2 index ab36c6518f5077..e93b08675e181d 100644 Binary files a/Tests/LibGfx/test-inputs/jbig2/bitmap-template1-tpgdon.jbig2 and b/Tests/LibGfx/test-inputs/jbig2/bitmap-template1-tpgdon.jbig2 differ diff --git a/Tests/LibGfx/test-inputs/jbig2/bitmap-template2-tpgdon.jbig2 b/Tests/LibGfx/test-inputs/jbig2/bitmap-template2-tpgdon.jbig2 index 6dbb070168d35b..0ed6f9561f2957 100644 Binary files a/Tests/LibGfx/test-inputs/jbig2/bitmap-template2-tpgdon.jbig2 and b/Tests/LibGfx/test-inputs/jbig2/bitmap-template2-tpgdon.jbig2 differ diff --git a/Tests/LibGfx/test-inputs/jbig2/bitmap-template3-tpgdon.jbig2 b/Tests/LibGfx/test-inputs/jbig2/bitmap-template3-tpgdon.jbig2 index c204b4dc1144e2..b457d348bc0bfb 100644 Binary files a/Tests/LibGfx/test-inputs/jbig2/bitmap-template3-tpgdon.jbig2 and b/Tests/LibGfx/test-inputs/jbig2/bitmap-template3-tpgdon.jbig2 differ diff --git a/Tests/LibGfx/test-inputs/jbig2/bitmap-tpgdon.jbig2 b/Tests/LibGfx/test-inputs/jbig2/bitmap-tpgdon.jbig2 index 6f42926eb64c0e..02d026dc382ea0 100644 Binary files a/Tests/LibGfx/test-inputs/jbig2/bitmap-tpgdon.jbig2 and b/Tests/LibGfx/test-inputs/jbig2/bitmap-tpgdon.jbig2 differ diff --git a/Tests/LibGfx/test-inputs/jbig2/json/bitmap-composite-and-xnor-halftone.json b/Tests/LibGfx/test-inputs/jbig2/json/bitmap-composite-and-xnor-halftone.json new file mode 100644 index 00000000000000..3db14fa6be6384 --- /dev/null +++ b/Tests/LibGfx/test-inputs/jbig2/json/bitmap-composite-and-xnor-halftone.json @@ -0,0 +1,241 @@ +{ + "global_header": { + "organization": "sequential", + "number_of_pages": 1 + }, + "segments": [ + { + "segment_number": 0, + "type": "page_information", + "page_association": 1, + "data": { + "page_width": 399, + "page_height": 400, + "flags": { + "default_color": "black", + "direct_region_segments_override_default_combination_operator": true + } + } + }, + + { + "segment_number": 1, + "type": "pattern_dictionary", + "page_association": 1, + "retained": true, + "data": { + "pattern_width": 16, + "pattern_height": 16, + "gray_max": "from_tiles", + "method": "unique_image_tiles", + "image_data": { + "from_file": "bitmap.bmp" + } + } + }, + + { + "segment_number": 2, + "type": "lossless_halftone_region", + "page_association": 1, + "referred_to_segments": [ + { + "segment_number": 1, + "retained": true + } + ], + "data": { + "region_segment_information": { + "x": 0, + "y": 0, + "width": 144, + "height": 400, + "flags": { + "external_combination_operator": "and" + } + }, + "grayscale_width": 9, + "grayscale_height": 25, + "grid_offset_x_times_256": 0, + "grid_offset_y_times_256": 0, + "grid_vector_x_times_256": 4096, + "grid_vector_y_times_256": 0, + "graymap_data": { + "match_image": { + "from_file": "bitmap.bmp", + "crop": { + "x": 0, + "y": 0, + "width": 144, + "height": 400 + } + } + } + } + }, + { + "segment_number": 3, + "type": "lossless_halftone_region", + "page_association": 1, + "referred_to_segments": [ + { + "segment_number": 1, + "retained": true + } + ], + "data": { + "region_segment_information": { + "x": 0, + "y": 0, + "width": 144, + "height": 400, + "flags": { + "external_combination_operator": "and" + } + }, + "grayscale_width": 9, + "grayscale_height": 25, + "grid_offset_x_times_256": 0, + "grid_offset_y_times_256": 0, + "grid_vector_x_times_256": 4096, + "grid_vector_y_times_256": 0, + "graymap_data": { + "match_image": { + "from_file": "bitmap.bmp", + "crop": { + "x": 0, + "y": 0, + "width": 144, + "height": 400 + } + } + } + } + }, + + { + "segment_number": 4, + "type": "lossless_halftone_region", + "page_association": 1, + "referred_to_segments": [ + { + "segment_number": 1, + "retained": true + } + ], + "data": { + "region_segment_information": { + "x": 144, + "y": 0, + "width": 255, + "height": 400, + "flags": { + "external_combination_operator": "xnor" + } + }, + "grayscale_width": 16, + "grayscale_height": 25, + "grid_offset_x_times_256": 0, + "grid_offset_y_times_256": 0, + "grid_vector_x_times_256": 4096, + "grid_vector_y_times_256": 0, + "graymap_data": { + "match_image": { + "from_file": "bitmap.bmp", + "crop": { + "x": 144, + "y": 0, + "width": 255, + "height": 400 + } + } + } + } + }, + + { + "segment_number": 5, + "type": "lossless_halftone_region", + "page_association": 1, + "referred_to_segments": [ + { + "segment_number": 1, + "retained": true + } + ], + "data": { + "region_segment_information": { + "x": 60, + "y": 192, + "width": 32, + "height": 32, + "flags": { + "external_combination_operator": "xnor" + } + }, + "grayscale_width": 2, + "grayscale_height": 2, + "grid_offset_x_times_256": 0, + "grid_offset_y_times_256": 0, + "grid_vector_x_times_256": 4096, + "grid_vector_y_times_256": 0, + "graymap_data": { + "match_image": { + "from_file": "bitmap.bmp", + "crop": { + "x": 60, + "y": 192, + "width": 32, + "height": 32 + } + } + } + } + }, + { + "segment_number": 6, + "type": "lossless_halftone_region", + "page_association": 1, + "referred_to_segments": [ + { + "segment_number": 1, + "retained": false + } + ], + "data": { + "region_segment_information": { + "x": 60, + "y": 192, + "width": 32, + "height": 32, + "flags": { + "external_combination_operator": "xnor" + } + }, + "grayscale_width": 2, + "grayscale_height": 2, + "grid_offset_x_times_256": 0, + "grid_offset_y_times_256": 0, + "grid_vector_x_times_256": 4096, + "grid_vector_y_times_256": 0, + "graymap_data": { + "match_image": { + "from_file": "bitmap.bmp", + "crop": { + "x": 60, + "y": 192, + "width": 32, + "height": 32 + } + } + } + } + }, + + { + "segment_number": 7, + "type": "end_of_page", + "page_association": 1 + } + ] +} diff --git a/Tests/LibGfx/test-inputs/jbig2/json/bitmap-composite-and-xnor-text.json b/Tests/LibGfx/test-inputs/jbig2/json/bitmap-composite-and-xnor-text.json new file mode 100644 index 00000000000000..0b18b6767e5e7a --- /dev/null +++ b/Tests/LibGfx/test-inputs/jbig2/json/bitmap-composite-and-xnor-text.json @@ -0,0 +1,322 @@ +{ + "global_header": { + "organization": "sequential", + "number_of_pages": 1 + }, + "segments": [ + { + "segment_number": 0, + "type": "page_information", + "page_association": 1, + "data": { + "page_width": 399, + "page_height": 400, + "flags": { + "default_color": "black", + "direct_region_segments_override_default_combination_operator": true + } + } + }, + + { + "segment_number": 1, + "type": "symbol_dictionary", + "page_association": 1, + "retained": true, + "data": { + "height_classes": [ + { + "symbols": [ + { + "image_data": { + "from_file": "bitmap.bmp", + "crop": { + "x": 60, + "y": 265, + "width": 35, + "height": 35 + } + } + }, + { + "image_data": { + "from_file": "bitmap.bmp", + "crop": { + "x": 50, + "y": 108, + "width": 35, + "height": 35 + } + } + } + ] + }, + { + "symbols": [ + { + "image_data": { + "from_file": "bitmap.bmp", + "crop": { + "x": 135, + "y": 100, + "width": 100, + "height": 100 + } + } + } + ] + }, + { + "symbols": [ + { + "image_data": { + "from_file": "bitmap.bmp", + "crop": { + "x": 232, + "y": 70, + "width": 120, + "height": 250 + } + } + } + ] + } + ] + } + }, + + { + "segment_number": 2, + "type": "lossless_text_region", + "page_association": 1, + "referred_to_segments": [ + { + "segment_number": 1, + "retained": true + } + ], + "data": { + "region_segment_information": { + "x": 0, + "y": 0, + "width": 141, + "height": 400, + "flags": { + "external_combination_operator": "and" + } + }, + "flags": { + "strip_size": 2, + "reference_corner": "top_left" + }, + "initial_strip_t": -2, + "strips": [ + { + "strip_t": 108, + "instances": [ + { + "instance_s": 50, + "instance_t": 108, + "symbol_id": 1 + } + ] + }, + { + "strip_t": 264, + "instances": [ + { + "instance_s": 60, + "instance_t": 265, + "symbol_id": 0 + } + ] + } + ] + } + }, + { + "segment_number": 3, + "type": "lossless_text_region", + "page_association": 1, + "referred_to_segments": [ + { + "segment_number": 1, + "retained": true + } + ], + "data": { + "region_segment_information": { + "x": 0, + "y": 0, + "width": 141, + "height": 400, + "flags": { + "external_combination_operator": "and" + } + }, + "flags": { + "strip_size": 2, + "reference_corner": "top_left" + }, + "initial_strip_t": -2, + "strips": [ + { + "strip_t": 108, + "instances": [ + { + "instance_s": 50, + "instance_t": 108, + "symbol_id": 1 + } + ] + }, + { + "strip_t": 264, + "instances": [ + { + "instance_s": 60, + "instance_t": 265, + "symbol_id": 0 + } + ] + } + ] + } + }, + + { + "segment_number": 4, + "type": "lossless_text_region", + "page_association": 1, + "referred_to_segments": [ + { + "segment_number": 1, + "retained": true + } + ], + "data": { + "region_segment_information": { + "x": 141, + "y": 0, + "width": 258, + "height": 400, + "flags": { + "external_combination_operator": "xnor" + } + }, + "flags": { + "strip_size": 2, + "reference_corner": "top_left" + }, + "initial_strip_t": -2, + "strips": [ + { + "strip_t": 70, + "instances": [ + { + "instance_s": 91, + "instance_t": 70, + "symbol_id": 3 + } + ] + }, + { + "strip_t": 100, + "instances": [ + { + "instance_s": -6, + "instance_t": 100, + "symbol_id": 2 + } + ] + } + ] + } + }, + + { + "segment_number": 5, + "type": "lossless_text_region", + "page_association": 1, + "referred_to_segments": [ + { + "segment_number": 1, + "retained": true + } + ], + "data": { + "region_segment_information": { + "x": 60, + "y": 190, + "width": 23, + "height": 23, + "flags": { + "external_combination_operator": "xnor" + } + }, + "flags": { + "strip_size": 2, + "reference_corner": "top_left" + }, + "initial_strip_t": -2, + "strips": [ + { + "strip_t": 0, + "instances": [ + { + "instance_s": 0, + "instance_t": 0, + "symbol_id": 0 + } + ] + } + ] + } + }, + { + "segment_number": 6, + "type": "lossless_text_region", + "page_association": 1, + "referred_to_segments": [ + { + "segment_number": 1, + "retained": false + } + ], + "data": { + "region_segment_information": { + "x": 60, + "y": 190, + "width": 23, + "height": 23, + "flags": { + "external_combination_operator": "xnor" + } + }, + "flags": { + "strip_size": 2, + "reference_corner": "top_left" + }, + "initial_strip_t": -2, + "strips": [ + { + "strip_t": 0, + "instances": [ + { + "instance_s": 0, + "instance_t": 0, + "symbol_id": 0 + } + ] + } + ] + } + }, + + { + "segment_number": 7, + "type": "end_of_page", + "page_association": 1 + } + ] +} diff --git a/Tests/LibGfx/test-inputs/jbig2/json/bitmap-composite-or-xor-replace-halftone.json b/Tests/LibGfx/test-inputs/jbig2/json/bitmap-composite-or-xor-replace-halftone.json new file mode 100644 index 00000000000000..3a9d18f9183ed9 --- /dev/null +++ b/Tests/LibGfx/test-inputs/jbig2/json/bitmap-composite-or-xor-replace-halftone.json @@ -0,0 +1,330 @@ +{ + "global_header": { + "organization": "sequential", + "number_of_pages": 1 + }, + "segments": [ + { + "segment_number": 0, + "type": "page_information", + "page_association": 1, + "data": { + "page_width": 399, + "page_height": 400, + "flags": { + "default_color": "white", + "direct_region_segments_override_default_combination_operator": true + } + } + }, + + { + "segment_number": 1, + "type": "pattern_dictionary", + "page_association": 1, + "retained": true, + "data": { + "pattern_width": 16, + "pattern_height": 16, + "gray_max": "from_tiles", + "method": "unique_image_tiles", + "image_data": { + "from_file": "bitmap.bmp" + } + } + }, + + { + "segment_number": 2, + "type": "lossless_halftone_region", + "page_association": 1, + "referred_to_segments": [ + { + "segment_number": 1, + "retained": true + } + ], + "data": { + "region_segment_information": { + "x": 48, + "y": 112, + "width": 32, + "height": 32, + "flags": { + "external_combination_operator": "or" + } + }, + "grayscale_width": 2, + "grayscale_height": 2, + "grid_offset_x_times_256": 0, + "grid_offset_y_times_256": 0, + "grid_vector_x_times_256": 4096, + "grid_vector_y_times_256": 0, + "graymap_data": { + "match_image": { + "from_file": "bitmap.bmp", + "crop": { + "x": 48, + "y": 112, + "width": 32, + "height": 32 + } + } + } + } + }, + { + "segment_number": 3, + "type": "lossless_halftone_region", + "page_association": 1, + "referred_to_segments": [ + { + "segment_number": 1, + "retained": true + } + ], + "data": { + "region_segment_information": { + "x": 48, + "y": 112, + "width": 32, + "height": 32, + "flags": { + "external_combination_operator": "or" + } + }, + "grayscale_width": 2, + "grayscale_height": 2, + "grid_offset_x_times_256": 0, + "grid_offset_y_times_256": 0, + "grid_vector_x_times_256": 4096, + "grid_vector_y_times_256": 0, + "graymap_data": { + "match_image": { + "from_file": "bitmap.bmp", + "crop": { + "x": 48, + "y": 112, + "width": 32, + "height": 32 + } + } + } + } + }, + { + "segment_number": 4, + "type": "lossless_halftone_region", + "page_association": 1, + "referred_to_segments": [ + { + "segment_number": 1, + "retained": true + } + ], + "data": { + "region_segment_information": { + "x": 48, + "y": 112, + "width": 32, + "height": 32, + "flags": { + "external_combination_operator": "or" + } + }, + "graymap_data": { + "array": [] + } + } + }, + + { + "segment_number": 5, + "type": "lossless_halftone_region", + "page_association": 1, + "referred_to_segments": [ + { + "segment_number": 1, + "retained": true + } + ], + "data": { + "region_segment_information": { + "x": 64, + "y": 256, + "width": 32, + "height": 48, + "flags": { + "external_combination_operator": "xor" + } + }, + "grayscale_width": 2, + "grayscale_height": 3, + "grid_offset_x_times_256": 0, + "grid_offset_y_times_256": 0, + "grid_vector_x_times_256": 4096, + "grid_vector_y_times_256": 0, + "graymap_data": { + "match_image": { + "from_file": "bitmap.bmp", + "crop": { + "x": 64, + "y": 256, + "width": 32, + "height": 48 + } + } + } + } + }, + + { + "segment_number": 6, + "type": "lossless_halftone_region", + "page_association": 1, + "referred_to_segments": [ + { + "segment_number": 1, + "retained": true + } + ], + "data": { + "region_segment_information": { + "x": 64, + "y": 192, + "width": 32, + "height": 32, + "flags": { + "external_combination_operator": "xor" + } + }, + "grayscale_width": 2, + "grayscale_height": 2, + "grid_offset_x_times_256": 0, + "grid_offset_y_times_256": 0, + "grid_vector_x_times_256": 4096, + "grid_vector_y_times_256": 0, + "graymap_data": { + "match_image": { + "from_file": "bitmap.bmp", + "crop": { + "x": 64, + "y": 192, + "width": 32, + "height": 32 + } + } + } + } + }, + { + "segment_number": 7, + "type": "lossless_halftone_region", + "page_association": 1, + "referred_to_segments": [ + { + "segment_number": 1, + "retained": true + } + ], + "data": { + "region_segment_information": { + "x": 64, + "y": 192, + "width": 32, + "height": 32, + "flags": { + "external_combination_operator": "xor" + } + }, + "grayscale_width": 2, + "grayscale_height": 2, + "grid_offset_x_times_256": 0, + "grid_offset_y_times_256": 0, + "grid_vector_x_times_256": 4096, + "grid_vector_y_times_256": 0, + "graymap_data": { + "match_image": { + "from_file": "bitmap.bmp", + "crop": { + "x": 64, + "y": 192, + "width": 32, + "height": 32 + } + } + } + } + }, + + { + "segment_number": 8, + "type": "lossless_generic_region", + "page_association": 1, + "data": { + "region_segment_information": { + "x": 128, + "y": 0, + "flags": { + "external_combination_operator": "or" + } + }, + "image_data": { + "from_file": "bitmap.bmp", + "crop": { + "x": 128, + "y": 0, + "width": 271, + "height": 400 + }, + "invert": true + } + } + }, + { + "segment_number": 9, + "type": "lossless_halftone_region", + "page_association": 1, + "referred_to_segments": [ + { + "segment_number": 1, + "retained": false + } + ], + "data": { + "region_segment_information": { + "x": 128, + "y": 0, + "width": 271, + "height": 400, + "flags": { + "external_combination_operator": "replace" + } + }, + "grayscale_width": 17, + "grayscale_height": 25, + "grid_offset_x_times_256": 0, + "grid_offset_y_times_256": 0, + "grid_vector_x_times_256": 4096, + "grid_vector_y_times_256": 0, + "graymap_data": { + "match_image": { + "from_file": "bitmap.bmp", + "crop": { + "x": 128, + "y": 0, + "width": 271, + "height": 400 + } + } + } + } + }, + + { + "segment_number": 10, + "type": "end_of_page", + "page_association": 1 + } + ] +} diff --git a/Tests/LibGfx/test-inputs/jbig2/json/bitmap-composite-or-xor-replace-refine.json b/Tests/LibGfx/test-inputs/jbig2/json/bitmap-composite-or-xor-replace-refine.json index d7af0823f8a052..3a9be28b8e4722 100644 --- a/Tests/LibGfx/test-inputs/jbig2/json/bitmap-composite-or-xor-replace-refine.json +++ b/Tests/LibGfx/test-inputs/jbig2/json/bitmap-composite-or-xor-replace-refine.json @@ -17,6 +17,7 @@ } } }, + { "segment_number": 1, "type": "intermediate_generic_region", @@ -188,6 +189,90 @@ "type": "intermediate_generic_region", "page_association": 1, "retained": true, + "data": { + "region_segment_information": { + "x": 55, + "y": 112, + "flags": { + "external_combination_operator": "or" + } + }, + "image_data": { + "from_file": "bitmap.bmp", + "crop": { + "x": 78, + "y": 112, + "width": 23, + "height": 23 + } + } + } + }, + { + "segment_number": 8, + "type": "intermediate_generic_refinement_region", + "page_association": 1, + "retained": true, + "referred_to_segments": [ + { + "segment_number": 7, + "retained": false + } + ], + "data": { + "region_segment_information": { + "x": 55, + "y": 112, + "flags": { + "external_combination_operator": "or" + } + }, + "image_data": { + "from_file": "bitmap.bmp", + "crop": { + "x": 78, + "y": 112, + "width": 23, + "height": 23 + } + } + } + }, + { + "segment_number": 9, + "type": "lossless_generic_refinement_region", + "page_association": 1, + "referred_to_segments": [ + { + "segment_number": 8, + "retained": false + } + ], + "data": { + "region_segment_information": { + "x": 55, + "y": 112, + "flags": { + "external_combination_operator": "or" + } + }, + "image_data": { + "from_file": "bitmap.bmp", + "crop": { + "x": 78, + "y": 112, + "width": 23, + "height": 23 + } + } + } + }, + + { + "segment_number": 10, + "type": "intermediate_generic_region", + "page_association": 1, + "retained": true, "data": { "region_segment_information": { "x": 64, @@ -208,13 +293,13 @@ } }, { - "segment_number": 8, + "segment_number": 11, "type": "intermediate_generic_refinement_region", "page_association": 1, "retained": true, "referred_to_segments": [ { - "segment_number": 7, + "segment_number": 10, "retained": false } ], @@ -238,12 +323,12 @@ } }, { - "segment_number": 9, + "segment_number": 12, "type": "generic_refinement_region", "page_association": 1, "referred_to_segments": [ { - "segment_number": 8, + "segment_number": 11, "retained": false } ], @@ -267,7 +352,7 @@ } }, { - "segment_number": 10, + "segment_number": 13, "type": "intermediate_generic_region", "page_association": 1, "retained": true, @@ -291,13 +376,13 @@ } }, { - "segment_number": 11, + "segment_number": 14, "type": "intermediate_generic_refinement_region", "page_association": 1, "retained": true, "referred_to_segments": [ { - "segment_number": 10, + "segment_number": 13, "retained": false } ], @@ -321,12 +406,12 @@ } }, { - "segment_number": 12, + "segment_number": 15, "type": "generic_refinement_region", "page_association": 1, "referred_to_segments": [ { - "segment_number": 11, + "segment_number": 14, "retained": false } ], @@ -350,7 +435,7 @@ } }, { - "segment_number": 13, + "segment_number": 16, "type": "intermediate_generic_region", "page_association": 1, "retained": true, @@ -374,13 +459,13 @@ } }, { - "segment_number": 14, + "segment_number": 17, "type": "intermediate_generic_refinement_region", "page_association": 1, "retained": true, "referred_to_segments": [ { - "segment_number": 13, + "segment_number": 16, "retained": false } ], @@ -404,12 +489,12 @@ } }, { - "segment_number": 15, + "segment_number": 18, "type": "lossless_generic_refinement_region", "page_association": 1, "referred_to_segments": [ { - "segment_number": 14, + "segment_number": 17, "retained": false } ], @@ -432,8 +517,9 @@ } } }, + { - "segment_number": 16, + "segment_number": 19, "type": "lossless_generic_region", "page_association": 1, "data": { @@ -457,7 +543,7 @@ } }, { - "segment_number": 17, + "segment_number": 20, "type": "intermediate_generic_region", "page_association": 1, "retained": true, @@ -481,13 +567,13 @@ } }, { - "segment_number": 18, + "segment_number": 21, "type": "intermediate_generic_refinement_region", "page_association": 1, "retained": true, "referred_to_segments": [ { - "segment_number": 17, + "segment_number": 20, "retained": false } ], @@ -511,12 +597,12 @@ } }, { - "segment_number": 19, + "segment_number": 22, "type": "generic_refinement_region", "page_association": 1, "referred_to_segments": [ { - "segment_number": 18, + "segment_number": 21, "retained": false } ], @@ -539,8 +625,9 @@ } } }, + { - "segment_number": 20, + "segment_number": 23, "type": "end_of_page", "page_association": 1 } diff --git a/Tests/LibGfx/test-inputs/jbig2/json/bitmap-composite-or-xor-replace-text.json b/Tests/LibGfx/test-inputs/jbig2/json/bitmap-composite-or-xor-replace-text.json new file mode 100644 index 00000000000000..df474c32ecd792 --- /dev/null +++ b/Tests/LibGfx/test-inputs/jbig2/json/bitmap-composite-or-xor-replace-text.json @@ -0,0 +1,394 @@ +{ + "global_header": { + "organization": "sequential", + "number_of_pages": 1 + }, + "segments": [ + { + "segment_number": 0, + "type": "page_information", + "page_association": 1, + "data": { + "page_width": 399, + "page_height": 400, + "flags": { + "default_color": "white", + "direct_region_segments_override_default_combination_operator": true + } + } + }, + + { + "segment_number": 1, + "type": "symbol_dictionary", + "page_association": 1, + "retained": true, + "data": { + "height_classes": [ + { + "symbols": [ + { + "image_data": { + "from_file": "bitmap.bmp", + "crop": { + "x": 60, + "y": 265, + "width": 35, + "height": 35 + } + } + }, + { + "image_data": { + "from_file": "bitmap.bmp", + "crop": { + "x": 50, + "y": 108, + "width": 35, + "height": 35 + } + } + } + ] + }, + { + "symbols": [ + { + "image_data": { + "from_file": "bitmap.bmp", + "crop": { + "x": 135, + "y": 100, + "width": 100, + "height": 100 + } + } + } + ] + }, + { + "symbols": [ + { + "image_data": { + "from_file": "bitmap.bmp", + "crop": { + "x": 232, + "y": 70, + "width": 120, + "height": 250 + } + } + } + ] + } + ] + } + }, + + { + "segment_number": 2, + "type": "lossless_text_region", + "page_association": 1, + "referred_to_segments": [ + { + "segment_number": 1, + "retained": true + } + ], + "data": { + "region_segment_information": { + "x": 55, + "y": 112, + "width": 35, + "height": 35, + "flags": { + "external_combination_operator": "or" + } + }, + "flags": { + "strip_size": 2, + "reference_corner": "top_left" + }, + "initial_strip_t": -4, + "strips": [ + { + "strip_t": -4, + "instances": [ + { + "instance_s": -5, + "instance_t": -4, + "symbol_id": 1 + } + ] + } + ] + } + }, + { + "segment_number": 3, + "type": "lossless_text_region", + "page_association": 1, + "referred_to_segments": [ + { + "segment_number": 1, + "retained": true + } + ], + "data": { + "region_segment_information": { + "x": 55, + "y": 112, + "width": 35, + "height": 35, + "flags": { + "external_combination_operator": "or" + } + }, + "flags": { + "strip_size": 2, + "reference_corner": "top_left" + }, + "initial_strip_t": -4, + "strips": [ + { + "strip_t": -4, + "instances": [ + { + "instance_s": -5, + "instance_t": -4, + "symbol_id": 1 + } + ] + } + ] + } + }, + { + "segment_number": 4, + "type": "lossless_text_region", + "page_association": 1, + "referred_to_segments": [ + { + "segment_number": 1, + "retained": true + } + ], + "data": { + "region_segment_information": { + "x": 55, + "y": 112, + "width": 35, + "height": 35, + "flags": { + "external_combination_operator": "or" + } + }, + "flags": { + "strip_size": 2 + }, + "initial_strip_t": -4, + "strips": [ + ] + } + }, + + { + "segment_number": 5, + "type": "lossless_text_region", + "page_association": 1, + "referred_to_segments": [ + { + "segment_number": 1, + "retained": true + } + ], + "data": { + "region_segment_information": { + "x": 64, + "y": 269, + "width": 35, + "height": 35, + "flags": { + "external_combination_operator": "xor" + } + }, + "flags": { + "strip_size": 2, + "reference_corner": "top_left" + }, + "initial_strip_t": -4, + "strips": [ + { + "strip_t": -4, + "instances": [ + { + "instance_s": -4, + "instance_t": -4, + "symbol_id": 0 + } + ] + } + ] + } + }, + + { + "segment_number": 6, + "type": "lossless_text_region", + "page_association": 1, + "referred_to_segments": [ + { + "segment_number": 1, + "retained": true + } + ], + "data": { + "region_segment_information": { + "x": 60, + "y": 190, + "width": 23, + "height": 23, + "flags": { + "external_combination_operator": "xor" + } + }, + "flags": { + "strip_size": 2, + "reference_corner": "top_left" + }, + "initial_strip_t": -2, + "strips": [ + { + "strip_t": 0, + "instances": [ + { + "instance_s": 0, + "instance_t": 0, + "symbol_id": 0 + } + ] + } + ] + } + }, + { + "segment_number": 7, + "type": "lossless_text_region", + "page_association": 1, + "referred_to_segments": [ + { + "segment_number": 1, + "retained": true + } + ], + "data": { + "region_segment_information": { + "x": 60, + "y": 190, + "width": 23, + "height": 23, + "flags": { + "external_combination_operator": "xor" + } + }, + "flags": { + "strip_size": 2, + "reference_corner": "top_left" + }, + "initial_strip_t": -2, + "strips": [ + { + "strip_t": 0, + "instances": [ + { + "instance_s": 0, + "instance_t": 0, + "symbol_id": 0 + } + ] + } + ] + } + }, + + { + "segment_number": 8, + "type": "lossless_generic_region", + "page_association": 1, + "data": { + "region_segment_information": { + "x": 141, + "y": 0, + "flags": { + "external_combination_operator": "or" + } + }, + "image_data": { + "from_file": "bitmap.bmp", + "crop": { + "x": 141, + "y": 0, + "width": 258, + "height": 400 + }, + "invert": true + } + } + }, + { + "segment_number": 9, + "type": "lossless_text_region", + "page_association": 1, + "referred_to_segments": [ + { + "segment_number": 1, + "retained": false + } + ], + "data": { + "region_segment_information": { + "x": 141, + "y": 0, + "width": 258, + "height": 400, + "flags": { + "external_combination_operator": "replace" + } + }, + "flags": { + "strip_size": 2, + "reference_corner": "top_left" + }, + "initial_strip_t": -2, + "strips": [ + { + "strip_t": 70, + "instances": [ + { + "instance_s": 91, + "instance_t": 70, + "symbol_id": 3 + } + ] + }, + { + "strip_t": 100, + "instances": [ + { + "instance_s": -6, + "instance_t": 100, + "symbol_id": 2 + } + ] + } + ] + } + }, + + { + "segment_number": 10, + "type": "end_of_page", + "page_association": 1 + } + ] +} diff --git a/Tests/LibGfx/test-inputs/jbig2/json/bitmap-composite-or-xor-replace.json b/Tests/LibGfx/test-inputs/jbig2/json/bitmap-composite-or-xor-replace.json index bd38d6da79e3a5..66ce23e222a73d 100644 --- a/Tests/LibGfx/test-inputs/jbig2/json/bitmap-composite-or-xor-replace.json +++ b/Tests/LibGfx/test-inputs/jbig2/json/bitmap-composite-or-xor-replace.json @@ -67,6 +67,29 @@ "segment_number": 3, "type": "lossless_generic_region", "page_association": 1, + "data": { + "region_segment_information": { + "x": 55, + "y": 112, + "flags": { + "external_combination_operator": "or" + } + }, + "image_data": { + "from_file": "bitmap.bmp", + "crop": { + "x": 80, + "y": 112, + "width": 23, + "height": 23 + } + } + } + }, + { + "segment_number": 4, + "type": "lossless_generic_region", + "page_association": 1, "data": { "region_segment_information": { "x": 64, @@ -87,7 +110,7 @@ } }, { - "segment_number": 4, + "segment_number": 5, "type": "lossless_generic_region", "page_association": 1, "data": { @@ -110,7 +133,7 @@ } }, { - "segment_number": 5, + "segment_number": 6, "type": "lossless_generic_region", "page_association": 1, "data": { @@ -133,7 +156,7 @@ } }, { - "segment_number": 6, + "segment_number": 7, "type": "lossless_generic_region", "page_association": 1, "data": { @@ -157,7 +180,7 @@ } }, { - "segment_number": 7, + "segment_number": 8, "type": "lossless_generic_region", "page_association": 1, "data": { @@ -180,7 +203,7 @@ } }, { - "segment_number": 8, + "segment_number": 9, "type": "end_of_page", "page_association": 1 } diff --git a/Tests/LibGfx/test-inputs/jbig2/json/bitmap-halftone-composite.json b/Tests/LibGfx/test-inputs/jbig2/json/bitmap-halftone-composite.json new file mode 100644 index 00000000000000..12749dbb5dc80f --- /dev/null +++ b/Tests/LibGfx/test-inputs/jbig2/json/bitmap-halftone-composite.json @@ -0,0 +1,358 @@ +{ + "global_header": { + "organization": "sequential", + "number_of_pages": 1 + }, + "segments": [ + { + "segment_number": 0, + "type": "page_information", + "page_association": 1, + "data": { + "page_width": 399, + "page_height": 400, + "flags": { + "is_eventually_lossless": true + } + } + }, + + { + "segment_number": 1, + "type": "pattern_dictionary", + "page_association": 1, + "retained": true, + "data": { + "pattern_width": 23, + "pattern_height": 23, + "gray_max": 1, + "image_data": [ + { + "from_file": "bitmap.bmp", + "crop": { + "x": 55, + "y": 112, + "width": 23, + "height": 23 + } + }, + { + "from_file": "bitmap.bmp", + "crop": { + "x": 78, + "y": 112, + "width": 23, + "height": 23 + } + } + ] + } + }, + { + "segment_number": 2, + "type": "lossless_halftone_region", + "page_association": 1, + "referred_to_segments": [ + { + "segment_number": 1, + "retained": false + } + ], + "data": { + "region_segment_information": { + "x": 55, + "y": 112, + "width": 23, + "height": 23 + }, + "flags": { + "combination_operator": "or", + "default_pixel_value": "white" + }, + "grayscale_width": 3, + "grayscale_height": 1, + "grid_offset_x_times_256": 0, + "grid_offset_y_times_256": 0, + "grid_vector_x_times_256": 0, + "grid_vector_y_times_256": 0, + "graymap_data": { + "array": [ + [ 0, 0, 1 ] + ] + } + } + }, + + { + "segment_number": 3, + "type": "pattern_dictionary", + "page_association": 1, + "retained": true, + "data": { + "pattern_width": 23, + "pattern_height": 23, + "gray_max": 1, + "image_data": [ + { + "from_file": "bitmap.bmp", + "crop": { + "x": 64, + "y": 269, + "width": 23, + "height": 23 + } + }, + { + "from_file": "bitmap.bmp", + "crop": { + "x": 190, + "y": 105, + "width": 23, + "height": 23 + } + } + ] + } + }, + { + "segment_number": 4, + "type": "lossless_halftone_region", + "page_association": 1, + "referred_to_segments": [ + { + "segment_number": 3, + "retained": false + } + ], + "data": { + "region_segment_information": { + "x": 64, + "y": 269, + "width": 23, + "height": 23 + }, + "flags": { + "combination_operator": "xor", + "default_pixel_value": "white" + }, + "grayscale_width": 3, + "grayscale_height": 1, + "grid_offset_x_times_256": 0, + "grid_offset_y_times_256": 0, + "grid_vector_x_times_256": 0, + "grid_vector_y_times_256": 0, + "graymap_data": { + "array": [ + [ 1, 0, 1 ] + ] + } + } + }, + + { + "segment_number": 5, + "type": "pattern_dictionary", + "page_association": 1, + "retained": true, + "data": { + "pattern_width": 100, + "pattern_height": 100, + "gray_max": 1, + "image_data": [ + { + "from_file": "bitmap.bmp", + "crop": { + "x": 135, + "y": 100, + "width": 100, + "height": 100 + } + }, + { + "from_file": "bitmap.bmp", + "crop": { + "x": 135, + "y": 100, + "width": 100, + "height": 100 + }, + "invert": true + } + ] + } + }, + { + "segment_number": 6, + "type": "lossless_halftone_region", + "page_association": 1, + "referred_to_segments": [ + { + "segment_number": 5, + "retained": false + } + ], + "data": { + "region_segment_information": { + "x": 135, + "y": 100, + "width": 100, + "height": 100 + }, + "flags": { + "combination_operator": "replace", + "default_pixel_value": "white" + }, + "grayscale_width": 2, + "grayscale_height": 1, + "grid_offset_x_times_256": 0, + "grid_offset_y_times_256": 0, + "grid_vector_x_times_256": 0, + "grid_vector_y_times_256": 0, + "graymap_data": { + "array": [ + [ 1, 0 ] + ] + } + } + }, + + { + "segment_number": 7, + "type": "pattern_dictionary", + "page_association": 1, + "retained": true, + "data": { + "pattern_width": 120, + "pattern_height": 130, + "gray_max": 1, + "image_data": [ + { + "from_file": "bitmap.bmp", + "crop": { + "x": 232, + "y": 70, + "width": 120, + "height": 130 + } + }, + { + "from_file": "bitmap.bmp", + "crop": { + "x": 100, + "y": 200, + "width": 120, + "height": 130 + }, + "invert": true + } + ] + } + }, + { + "segment_number": 8, + "type": "lossless_halftone_region", + "page_association": 1, + "referred_to_segments": [ + { + "segment_number": 7, + "retained": false + } + ], + "data": { + "region_segment_information": { + "x": 232, + "y": 70, + "width": 120, + "height": 130 + }, + "flags": { + "combination_operator": "and", + "default_pixel_value": "black" + }, + "grayscale_width": 3, + "grayscale_height": 1, + "grid_offset_x_times_256": 0, + "grid_offset_y_times_256": 0, + "grid_vector_x_times_256": 0, + "grid_vector_y_times_256": 0, + "graymap_data": { + "array": [ + [ 0, 0, 1 ] + ] + } + } + }, + + { + "segment_number": 9, + "type": "pattern_dictionary", + "page_association": 1, + "retained": true, + "data": { + "pattern_width": 120, + "pattern_height": 120, + "gray_max": 1, + "image_data": [ + { + "from_file": "bitmap.bmp", + "crop": { + "x": 232, + "y": 200, + "width": 120, + "height": 120 + } + }, + { + "from_file": "bitmap.bmp", + "crop": { + "x": 232, + "y": 70, + "width": 120, + "height": 120 + } + } + ] + } + }, + { + "segment_number": 10, + "type": "lossless_halftone_region", + "page_association": 1, + "referred_to_segments": [ + { + "segment_number": 9, + "retained": false + } + ], + "data": { + "region_segment_information": { + "x": 232, + "y": 200, + "width": 120, + "height": 120 + }, + "flags": { + "combination_operator": "xnor", + "default_pixel_value": "black" + }, + "grayscale_width": 3, + "grayscale_height": 1, + "grid_offset_x_times_256": 0, + "grid_offset_y_times_256": 0, + "grid_vector_x_times_256": 0, + "grid_vector_y_times_256": 0, + "graymap_data": { + "array": [ + [ 1, 0, 1 ] + ] + } + } + }, + + { + "segment_number": 11, + "type": "end_of_page", + "page_association": 1 + } + ] +} diff --git a/Tests/LibGfx/test-inputs/jbig2/json/bitmap-halftone-grid.json b/Tests/LibGfx/test-inputs/jbig2/json/bitmap-halftone-grid.json new file mode 100644 index 00000000000000..fe2ea253cf94f8 --- /dev/null +++ b/Tests/LibGfx/test-inputs/jbig2/json/bitmap-halftone-grid.json @@ -0,0 +1,76 @@ +{ + "global_header": { + "organization": "sequential", + "number_of_pages": 1 + }, + "segments": [ + { + "segment_number": 0, + "type": "page_information", + "page_association": 1, + "data": { + "page_width": 399, + "page_height": 400, + "flags": { + "is_eventually_lossless": true + } + } + }, + { + "segment_number": 1, + "type": "pattern_dictionary", + "page_association": 1, + "retained": true, + "data": { + "pattern_width": 16, + "pattern_height": 16, + "gray_max": "from_tiles", + "method": "unique_image_tiles", + "image_data": { + "from_file": "bitmap.bmp" + }, + "grayscale_width": 29, + "grayscale_height": 29, + "grid_offset_x_times_256": -8192, + "grid_offset_y_times_256": 20000, + "grid_vector_x_times_256": 3072, + "grid_vector_y_times_256": 1024 + } + }, + { + "segment_number": 2, + "type": "lossless_halftone_region", + "page_association": 1, + "referred_to_segments": [ + { + "segment_number": 1, + "retained": false + } + ], + "data": { + "region_segment_information": { + "width": 399, + "height": 400 + }, + "flags": { + "combination_operator": "or", + "default_pixel_value": "white" + }, + "grayscale_width": 29, + "grayscale_height": 29, + "grid_offset_x_times_256": -8192, + "grid_offset_y_times_256": 20000, + "grid_vector_x_times_256": 3072, + "grid_vector_y_times_256": 1024, + "graymap_data": { + "match_image": "bitmap.bmp" + } + } + }, + { + "segment_number": 3, + "type": "end_of_page", + "page_association": 1 + } + ] +} diff --git a/Tests/LibGfx/test-inputs/jbig2/json/bitmap-halftone-refine.json b/Tests/LibGfx/test-inputs/jbig2/json/bitmap-halftone-refine.json new file mode 100644 index 00000000000000..510575af1b4508 --- /dev/null +++ b/Tests/LibGfx/test-inputs/jbig2/json/bitmap-halftone-refine.json @@ -0,0 +1,83 @@ +{ + "global_header": { + "organization": "sequential", + "number_of_pages": 1 + }, + "segments": [ + { + "segment_number": 0, + "type": "page_information", + "page_association": 1, + "data": { + "page_width": 399, + "page_height": 400, + "flags": { + "is_eventually_lossless": true + } + } + }, + { + "segment_number": 1, + "type": "pattern_dictionary", + "page_association": 1, + "retained": true, + "data": { + "pattern_width": 16, + "pattern_height": 16, + "gray_max": "from_tiles", + "method": "unique_image_tiles", + "image_data": { + "from_file": "bitmap-blemish.bmp" + } + } + }, + { + "segment_number": 2, + "type": "intermediate_halftone_region", + "page_association": 1, + "retained": true, + "referred_to_segments": [ + { + "segment_number": 1, + "retained": false + } + ], + "data": { + "region_segment_information": { + "width": 399, + "height": 400 + }, + "grayscale_width": 25, + "grayscale_height": 25, + "grid_offset_x_times_256": 0, + "grid_offset_y_times_256": 0, + "grid_vector_x_times_256": 4096, + "grid_vector_y_times_256": 0, + "graymap_data": { + "match_image": "bitmap-blemish.bmp" + } + } + }, + { + "segment_number": 3, + "type": "generic_refinement_region", + "page_association": 1, + "referred_to_segments": [ + { + "segment_number": 2, + "retained": false + } + ], + "data": { + "image_data": { + "from_file": "bitmap.bmp" + } + } + }, + { + "segment_number": 4, + "type": "end_of_page", + "page_association": 1 + } + ] +} diff --git a/Tests/LibGfx/test-inputs/jbig2/json/bitmap-halftone-skip-dummy.json b/Tests/LibGfx/test-inputs/jbig2/json/bitmap-halftone-skip-dummy.json new file mode 100644 index 00000000000000..096fe65d64f21a --- /dev/null +++ b/Tests/LibGfx/test-inputs/jbig2/json/bitmap-halftone-skip-dummy.json @@ -0,0 +1,69 @@ +{ + "global_header": { + "organization": "sequential", + "number_of_pages": 1 + }, + "segments": [ + { + "segment_number": 0, + "type": "page_information", + "page_association": 1, + "data": { + "page_width": 399, + "page_height": 400, + "flags": { + "is_eventually_lossless": true + } + } + }, + { + "segment_number": 1, + "type": "pattern_dictionary", + "page_association": 1, + "retained": true, + "data": { + "pattern_width": 16, + "pattern_height": 16, + "gray_max": "from_tiles", + "method": "unique_image_tiles", + "image_data": { + "from_file": "bitmap.bmp" + } + } + }, + { + "segment_number": 2, + "type": "lossless_halftone_region", + "page_association": 1, + "referred_to_segments": [ + { + "segment_number": 1, + "retained": false + } + ], + "data": { + "region_segment_information": { + "width": 399, + "height": 400 + }, + "flags": { + "enable_skip": true + }, + "grayscale_width": 25, + "grayscale_height": 25, + "grid_offset_x_times_256": 0, + "grid_offset_y_times_256": 0, + "grid_vector_x_times_256": 4096, + "grid_vector_y_times_256": 0, + "graymap_data": { + "match_image": "bitmap.bmp" + } + } + }, + { + "segment_number": 3, + "type": "end_of_page", + "page_association": 1 + } + ] +} diff --git a/Tests/LibGfx/test-inputs/jbig2/json/bitmap-halftone-skip-grid-template1.json b/Tests/LibGfx/test-inputs/jbig2/json/bitmap-halftone-skip-grid-template1.json new file mode 100644 index 00000000000000..a5fec875b74960 --- /dev/null +++ b/Tests/LibGfx/test-inputs/jbig2/json/bitmap-halftone-skip-grid-template1.json @@ -0,0 +1,78 @@ +{ + "global_header": { + "organization": "sequential", + "number_of_pages": 1 + }, + "segments": [ + { + "segment_number": 0, + "type": "page_information", + "page_association": 1, + "data": { + "page_width": 399, + "page_height": 400, + "flags": { + "is_eventually_lossless": true + } + } + }, + { + "segment_number": 1, + "type": "pattern_dictionary", + "page_association": 1, + "retained": true, + "data": { + "pattern_width": 16, + "pattern_height": 16, + "gray_max": "from_tiles", + "method": "unique_image_tiles", + "image_data": { + "from_file": "bitmap.bmp" + }, + "grayscale_width": 43, + "grayscale_height": 41, + "grid_offset_x_times_256": -29160, + "grid_offset_y_times_256": 10424, + "grid_vector_x_times_256": 3072, + "grid_vector_y_times_256": 1024 + } + }, + { + "segment_number": 2, + "type": "lossless_halftone_region", + "page_association": 1, + "referred_to_segments": [ + { + "segment_number": 1, + "retained": false + } + ], + "data": { + "region_segment_information": { + "width": 399, + "height": 400 + }, + "flags": { + "ht_template": 1, + "enable_skip": true, + "combination_operator": "or", + "default_pixel_value": "white" + }, + "grayscale_width": 43, + "grayscale_height": 41, + "grid_offset_x_times_256": -29160, + "grid_offset_y_times_256": 10424, + "grid_vector_x_times_256": 3072, + "grid_vector_y_times_256": 1024, + "graymap_data": { + "match_image": "bitmap.bmp" + } + } + }, + { + "segment_number": 3, + "type": "end_of_page", + "page_association": 1 + } + ] +} diff --git a/Tests/LibGfx/test-inputs/jbig2/json/bitmap-halftone-skip-grid-template2.json b/Tests/LibGfx/test-inputs/jbig2/json/bitmap-halftone-skip-grid-template2.json new file mode 100644 index 00000000000000..bc461ebb7bde2b --- /dev/null +++ b/Tests/LibGfx/test-inputs/jbig2/json/bitmap-halftone-skip-grid-template2.json @@ -0,0 +1,78 @@ +{ + "global_header": { + "organization": "sequential", + "number_of_pages": 1 + }, + "segments": [ + { + "segment_number": 0, + "type": "page_information", + "page_association": 1, + "data": { + "page_width": 399, + "page_height": 400, + "flags": { + "is_eventually_lossless": true + } + } + }, + { + "segment_number": 1, + "type": "pattern_dictionary", + "page_association": 1, + "retained": true, + "data": { + "pattern_width": 16, + "pattern_height": 16, + "gray_max": "from_tiles", + "method": "unique_image_tiles", + "image_data": { + "from_file": "bitmap.bmp" + }, + "grayscale_width": 43, + "grayscale_height": 41, + "grid_offset_x_times_256": -29160, + "grid_offset_y_times_256": 10424, + "grid_vector_x_times_256": 3072, + "grid_vector_y_times_256": 1024 + } + }, + { + "segment_number": 2, + "type": "lossless_halftone_region", + "page_association": 1, + "referred_to_segments": [ + { + "segment_number": 1, + "retained": false + } + ], + "data": { + "region_segment_information": { + "width": 399, + "height": 400 + }, + "flags": { + "ht_template": 2, + "enable_skip": true, + "combination_operator": "or", + "default_pixel_value": "white" + }, + "grayscale_width": 43, + "grayscale_height": 41, + "grid_offset_x_times_256": -29160, + "grid_offset_y_times_256": 10424, + "grid_vector_x_times_256": 3072, + "grid_vector_y_times_256": 1024, + "graymap_data": { + "match_image": "bitmap.bmp" + } + } + }, + { + "segment_number": 3, + "type": "end_of_page", + "page_association": 1 + } + ] +} diff --git a/Tests/LibGfx/test-inputs/jbig2/json/bitmap-halftone-skip-grid-template3.json b/Tests/LibGfx/test-inputs/jbig2/json/bitmap-halftone-skip-grid-template3.json new file mode 100644 index 00000000000000..4c4ea46c2545c3 --- /dev/null +++ b/Tests/LibGfx/test-inputs/jbig2/json/bitmap-halftone-skip-grid-template3.json @@ -0,0 +1,78 @@ +{ + "global_header": { + "organization": "sequential", + "number_of_pages": 1 + }, + "segments": [ + { + "segment_number": 0, + "type": "page_information", + "page_association": 1, + "data": { + "page_width": 399, + "page_height": 400, + "flags": { + "is_eventually_lossless": true + } + } + }, + { + "segment_number": 1, + "type": "pattern_dictionary", + "page_association": 1, + "retained": true, + "data": { + "pattern_width": 16, + "pattern_height": 16, + "gray_max": "from_tiles", + "method": "unique_image_tiles", + "image_data": { + "from_file": "bitmap.bmp" + }, + "grayscale_width": 43, + "grayscale_height": 41, + "grid_offset_x_times_256": -29160, + "grid_offset_y_times_256": 10424, + "grid_vector_x_times_256": 3072, + "grid_vector_y_times_256": 1024 + } + }, + { + "segment_number": 2, + "type": "lossless_halftone_region", + "page_association": 1, + "referred_to_segments": [ + { + "segment_number": 1, + "retained": false + } + ], + "data": { + "region_segment_information": { + "width": 399, + "height": 400 + }, + "flags": { + "ht_template": 3, + "enable_skip": true, + "combination_operator": "or", + "default_pixel_value": "white" + }, + "grayscale_width": 43, + "grayscale_height": 41, + "grid_offset_x_times_256": -29160, + "grid_offset_y_times_256": 10424, + "grid_vector_x_times_256": 3072, + "grid_vector_y_times_256": 1024, + "graymap_data": { + "match_image": "bitmap.bmp" + } + } + }, + { + "segment_number": 3, + "type": "end_of_page", + "page_association": 1 + } + ] +} diff --git a/Tests/LibGfx/test-inputs/jbig2/json/bitmap-halftone-skip-grid.json b/Tests/LibGfx/test-inputs/jbig2/json/bitmap-halftone-skip-grid.json new file mode 100644 index 00000000000000..76dfd2b52b4819 --- /dev/null +++ b/Tests/LibGfx/test-inputs/jbig2/json/bitmap-halftone-skip-grid.json @@ -0,0 +1,77 @@ +{ + "global_header": { + "organization": "sequential", + "number_of_pages": 1 + }, + "segments": [ + { + "segment_number": 0, + "type": "page_information", + "page_association": 1, + "data": { + "page_width": 399, + "page_height": 400, + "flags": { + "is_eventually_lossless": true + } + } + }, + { + "segment_number": 1, + "type": "pattern_dictionary", + "page_association": 1, + "retained": true, + "data": { + "pattern_width": 16, + "pattern_height": 16, + "gray_max": "from_tiles", + "method": "unique_image_tiles", + "image_data": { + "from_file": "bitmap.bmp" + }, + "grayscale_width": 43, + "grayscale_height": 41, + "grid_offset_x_times_256": -29160, + "grid_offset_y_times_256": 10424, + "grid_vector_x_times_256": 3072, + "grid_vector_y_times_256": 1024 + } + }, + { + "segment_number": 2, + "type": "lossless_halftone_region", + "page_association": 1, + "referred_to_segments": [ + { + "segment_number": 1, + "retained": false + } + ], + "data": { + "region_segment_information": { + "width": 399, + "height": 400 + }, + "flags": { + "enable_skip": true, + "combination_operator": "or", + "default_pixel_value": "white" + }, + "grayscale_width": 43, + "grayscale_height": 41, + "grid_offset_x_times_256": -29160, + "grid_offset_y_times_256": 10424, + "grid_vector_x_times_256": 3072, + "grid_vector_y_times_256": 1024, + "graymap_data": { + "match_image": "bitmap.bmp" + } + } + }, + { + "segment_number": 3, + "type": "end_of_page", + "page_association": 1 + } + ] +} diff --git a/Tests/LibGfx/test-inputs/jbig2/json/bitmap-randomaccess.json b/Tests/LibGfx/test-inputs/jbig2/json/bitmap-randomaccess.json new file mode 100644 index 00000000000000..e60f95902344f7 --- /dev/null +++ b/Tests/LibGfx/test-inputs/jbig2/json/bitmap-randomaccess.json @@ -0,0 +1,40 @@ +{ + "global_header": { + "organization": "random_access", + "number_of_pages": 1 + }, + "segments": [ + { + "segment_number": 0, + "type": "page_information", + "page_association": 1, + "data": { + "page_width": 399, + "page_height": 400, + "flags": { + "is_eventually_lossless": true + } + } + }, + { + "segment_number": 1, + "type": "lossless_generic_region", + "page_association": 1, + "data": { + "image_data": { + "from_file": "bitmap.bmp" + } + } + }, + { + "segment_number": 2, + "type": "end_of_page", + "page_association": 1 + }, + { + "segment_number": 3, + "type": "end_of_file", + "page_association": 0 + } + ] +} diff --git a/Tests/LibGfx/test-inputs/jbig2/json/bitmap-refine-page-subrect.json b/Tests/LibGfx/test-inputs/jbig2/json/bitmap-refine-page-subrect.json new file mode 100644 index 00000000000000..0a04f558103532 --- /dev/null +++ b/Tests/LibGfx/test-inputs/jbig2/json/bitmap-refine-page-subrect.json @@ -0,0 +1,60 @@ +{ + "global_header": { + "organization": "sequential", + "number_of_pages": 1 + }, + "segments": [ + { + "segment_number": 0, + "type": "page_information", + "page_association": 1, + "data": { + "page_width": 399, + "page_height": 400, + "flags": { + "direct_region_segments_override_default_combination_operator": true + } + } + }, + { + "segment_number": 1, + "type": "generic_region", + "page_association": 1, + "data": { + "image_data": { + "from_file": "bitmap-blemish.bmp" + } + } + }, + { + "segment_number": 2, + "type": "generic_refinement_region", + "page_association": 1, + "data": { + "region_segment_information": { + "x": 10, + "y": 20, + "width": 110, + "height": 380, + "flags": { + "external_combination_operator": "replace" + } + }, + "image_data": { + "from_file": "bitmap.bmp", + "crop": { + "x": 10, + "y": 20, + "width": 110, + "height": 380 + } + } + } + }, + { + "segment_number": 3, + "type": "end_of_page", + "page_association": 1 + } + ] +} diff --git a/Tests/LibGfx/test-inputs/jbig2/json/bitmap-refine-page.json b/Tests/LibGfx/test-inputs/jbig2/json/bitmap-refine-page.json new file mode 100644 index 00000000000000..f9781934924fae --- /dev/null +++ b/Tests/LibGfx/test-inputs/jbig2/json/bitmap-refine-page.json @@ -0,0 +1,50 @@ +{ + "global_header": { + "organization": "sequential", + "number_of_pages": 1 + }, + "segments": [ + { + "segment_number": 0, + "type": "page_information", + "page_association": 1, + "data": { + "page_width": 399, + "page_height": 400, + "flags": { + "direct_region_segments_override_default_combination_operator": true + } + } + }, + { + "segment_number": 1, + "type": "generic_region", + "page_association": 1, + "data": { + "image_data": { + "from_file": "bitmap-blemish.bmp" + } + } + }, + { + "segment_number": 2, + "type": "generic_refinement_region", + "page_association": 1, + "data": { + "region_segment_information": { + "flags": { + "external_combination_operator": "replace" + } + }, + "image_data": { + "from_file": "bitmap.bmp" + } + } + }, + { + "segment_number": 3, + "type": "end_of_page", + "page_association": 1 + } + ] +} diff --git a/Tests/LibGfx/test-inputs/jbig2/json/bitmap-refine-template1-tpgron.json b/Tests/LibGfx/test-inputs/jbig2/json/bitmap-refine-template1-tpgron.json index 4124c5dbb4c871..99067e747beaf4 100644 --- a/Tests/LibGfx/test-inputs/jbig2/json/bitmap-refine-template1-tpgron.json +++ b/Tests/LibGfx/test-inputs/jbig2/json/bitmap-refine-template1-tpgron.json @@ -46,6 +46,43 @@ }, { "segment_number": 3, + "type": "intermediate_generic_region", + "page_association": 1, + "retained": true, + "data": { + "image_data": { + "from_file": "tpgron-ref.bmp", + "repeat_x": 66, + "repeat_y": 33 + } + } + }, + { + "segment_number": 4, + "type": "generic_refinement_region", + "page_association": 1, + "referred_to_segments": [ + { + "segment_number": 3, + "retained": true + } + ], + "data": { + "image_data": { + "from_file": "bitmap.bmp", + "crop": { + "width": 198, + "height": 99 + } + }, + "flags": { + "gr_template": 1, + "use_typical_prediction": true + } + } + }, + { + "segment_number": 5, "type": "end_of_page", "page_association": 1 } diff --git a/Tests/LibGfx/test-inputs/jbig2/json/bitmap-refine-tpgron.json b/Tests/LibGfx/test-inputs/jbig2/json/bitmap-refine-tpgron.json index c57897e0f336bd..bca2c9ab44ece1 100644 --- a/Tests/LibGfx/test-inputs/jbig2/json/bitmap-refine-tpgron.json +++ b/Tests/LibGfx/test-inputs/jbig2/json/bitmap-refine-tpgron.json @@ -45,6 +45,42 @@ }, { "segment_number": 3, + "type": "intermediate_generic_region", + "page_association": 1, + "retained": true, + "data": { + "image_data": { + "from_file": "tpgron-ref.bmp", + "repeat_x": 66, + "repeat_y": 33 + } + } + }, + { + "segment_number": 4, + "type": "generic_refinement_region", + "page_association": 1, + "referred_to_segments": [ + { + "segment_number": 3, + "retained": true + } + ], + "data": { + "image_data": { + "from_file": "bitmap.bmp", + "crop": { + "width": 198, + "height": 99 + } + }, + "flags": { + "use_typical_prediction": true + } + } + }, + { + "segment_number": 5, "type": "end_of_page", "page_association": 1 } diff --git a/Tests/LibGfx/test-inputs/jbig2/json/bitmap-symbol-big-segmentid.json b/Tests/LibGfx/test-inputs/jbig2/json/bitmap-symbol-big-segmentid.json new file mode 100644 index 00000000000000..284522358c4b0d --- /dev/null +++ b/Tests/LibGfx/test-inputs/jbig2/json/bitmap-symbol-big-segmentid.json @@ -0,0 +1,150 @@ +{ + "global_header": { + "organization": "sequential", + "number_of_pages": 1 + }, + "segments": [ + { + "segment_number": 0, + "type": "page_information", + "page_association": 1, + "data": { + "page_width": 399, + "page_height": 400, + "flags": { + "is_eventually_lossless": true + } + } + }, + + { + "segment_number": 256, + "type": "symbol_dictionary", + "page_association": 1, + "retained": true, + "data": { + "height_classes": [ + { + "symbols": [ + { + "image_data": { + "from_file": "bitmap.bmp", + "crop": { + "x": 0, + "y": 0, + "width": 399, + "height": 200 + } + } + } + ] + } + ] + } + }, + { + "segment_number": 257, + "type": "lossless_text_region", + "page_association": 1, + "referred_to_segments": [ + { + "segment_number": 256, + "retained": false + } + ], + "data": { + "region_segment_information": { + "x": 0, + "y": 0, + "width": 399, + "height": 200 + }, + "flags": { + "strip_size": 4, + "reference_corner": "top_left" + }, + "initial_strip_t": -4, + "strips": [ + { + "strip_t": 0, + "instances": [ + { + "instance_s": 0, + "instance_t": 0, + "symbol_id": 0 + } + ] + } + ] + } + }, + + { + "segment_number": 65536, + "type": "symbol_dictionary", + "page_association": 1, + "retained": true, + "data": { + "height_classes": [ + { + "symbols": [ + { + "image_data": { + "from_file": "bitmap.bmp", + "crop": { + "x": 0, + "y": 200, + "width": 399, + "height": 200 + } + } + } + ] + } + ] + } + }, + { + "segment_number": 65537, + "type": "lossless_text_region", + "page_association": 1, + "referred_to_segments": [ + { + "segment_number": 65536, + "retained": false + } + ], + "data": { + "region_segment_information": { + "x": 0, + "y": 200, + "width": 399, + "height": 200 + }, + "flags": { + "strip_size": 4, + "reference_corner": "top_left" + }, + "initial_strip_t": -4, + "strips": [ + { + "strip_t": 0, + "instances": [ + { + "instance_s": 0, + "instance_t": 0, + "symbol_id": 0 + } + ] + } + ] + } + }, + + { + "segment_number": 300000, + "type": "end_of_page", + "page_association": 1 + } + ] +} diff --git a/Tests/LibGfx/test-inputs/jbig2/json/bitmap-symbol-context-reuse.json b/Tests/LibGfx/test-inputs/jbig2/json/bitmap-symbol-context-reuse.json new file mode 100644 index 00000000000000..f09aa4c14de038 --- /dev/null +++ b/Tests/LibGfx/test-inputs/jbig2/json/bitmap-symbol-context-reuse.json @@ -0,0 +1,262 @@ +{ + "global_header": { + "organization": "sequential", + "number_of_pages": 1 + }, + "segments": [ + { + "segment_number": 0, + "type": "page_information", + "page_association": 1, + "data": { + "page_width": 399, + "page_height": 400, + "flags": { + "is_eventually_lossless": true + } + } + }, + + { + "segment_number": 1, + "type": "symbol_dictionary", + "page_association": 1, + "retained": true, + "data": { + "flags": { + "uses_huffman_encoding": false, + "uses_refinement_or_aggregate_coding": false, + "is_bitmap_coding_context_used": false, + "is_bitmap_coding_context_retained": true, + "template": 0, + "refinement_template": 0 + }, + "height_classes": [ + { + "symbols": [ + { + "image_data": { + "from_file": "bitmap.bmp", + "crop": { + "x": 50, + "y": 108, + "width": 30, + "height": 30 + } + } + } + ] + } + ] + } + }, + + { + "segment_number": 2, + "type": "symbol_dictionary", + "page_association": 1, + "retained": true, + "referred_to_segments": [ + { + "segment_number": 1, + "retained": true + } + ], + "data": { + "flags": { + "uses_huffman_encoding": false, + "uses_refinement_or_aggregate_coding": false, + "is_bitmap_coding_context_used": true, + "is_bitmap_coding_context_retained": false, + "template": 0, + "refinement_template": 0 + }, + "height_classes": [ + { + "symbols": [ + { + "image_data": { + "from_file": "bitmap.bmp", + "crop": { + "x": 60, + "y": 265, + "width": 30, + "height": 30 + } + } + } + ] + } + ], + "export_flags_for_referred_to_symbols": [ false ] + } + }, + + { + "segment_number": 3, + "type": "symbol_dictionary", + "page_association": 1, + "retained": true, + "referred_to_segments": [ + { + "segment_number": 1, + "retained": true + } + ], + "data": { + "flags": { + "uses_huffman_encoding": false, + "uses_refinement_or_aggregate_coding": false, + "is_bitmap_coding_context_used": true, + "is_bitmap_coding_context_retained": true, + "template": 0, + "refinement_template": 0 + }, + "height_classes": [ + { + "symbols": [ + { + "image_data": { + "from_file": "bitmap.bmp", + "crop": { + "x": 135, + "y": 100, + "width": 100, + "height": 100 + } + } + } + ] + } + ], + "export_flags_for_referred_to_symbols": [ false ] + } + }, + + { + "segment_number": 4, + "type": "symbol_dictionary", + "page_association": 1, + "retained": true, + "referred_to_segments": [ + { + "segment_number": 1, + "retained": false + }, + { + "segment_number": 2, + "retained": false + }, + { + "segment_number": 3, + "retained": false + } + ], + "data": { + "flags": { + "uses_huffman_encoding": false, + "uses_refinement_or_aggregate_coding": false, + "is_bitmap_coding_context_used": true, + "is_bitmap_coding_context_retained": false, + "template": 0, + "refinement_template": 0 + }, + "height_classes": [ + { + "symbols": [ + { + "image_data": { + "from_file": "bitmap.bmp", + "crop": { + "x": 232, + "y": 70, + "width": 120, + "height": 250 + } + } + } + ] + } + ] + } + }, + + { + "segment_number": 5, + "type": "lossless_text_region", + "page_association": 1, + "referred_to_segments": [ + { + "segment_number": 4, + "retained": false + } + ], + "data": { + "region_segment_information": { + "width": 399, + "height": 400 + }, + "flags": { + "uses_huffman_encoding": false, + "uses_refinement_coding": false, + "strip_size": 4, + "reference_corner": "top_left", + "is_transposed": false, + "combination_operator": "or", + "default_pixel_value": "white", + "delta_s_offset": 0 + }, + + "initial_strip_t": -4, + "strips": [ + { + "strip_t": 68, + "instances": [ + { + "instance_s": 232, + "instance_t": 70, + "symbol_id": 3 + } + ] + }, + { + "strip_t": 100, + "instances": [ + { + "instance_s": 135, + "instance_t": 100, + "symbol_id": 2 + } + ] + }, + { + "strip_t": 108, + "instances": [ + { + "instance_s": 50, + "instance_t": 108, + "symbol_id": 0 + } + ] + }, + { + "strip_t": 264, + "instances": [ + { + "instance_s": 60, + "instance_t": 265, + "symbol_id": 1 + } + ] + } + ] + } + }, + + { + "segment_number": 6, + "type": "end_of_page", + "page_association": 1 + } + ] +} diff --git a/Tests/LibGfx/test-inputs/jbig2/json/bitmap-symbol-empty.json b/Tests/LibGfx/test-inputs/jbig2/json/bitmap-symbol-empty.json new file mode 100644 index 00000000000000..d9bd0bd73c58ce --- /dev/null +++ b/Tests/LibGfx/test-inputs/jbig2/json/bitmap-symbol-empty.json @@ -0,0 +1,95 @@ +{ + "global_header": { + "organization": "sequential", + "number_of_pages": 1 + }, + "segments": [ + { + "segment_number": 0, + "type": "page_information", + "page_association": 1, + "data": { + "page_width": 399, + "page_height": 400, + "flags": { + "is_eventually_lossless": true + } + } + }, + + { + "segment_number": 1, + "type": "symbol_dictionary", + "page_association": 1, + "retained": true, + "data": { + "height_classes": [ + ] + } + }, + { + "segment_number": 2, + "type": "lossless_text_region", + "page_association": 1, + "referred_to_segments": [ + { + "segment_number": 1, + "retained": false + } + ], + "data": { + "region_segment_information": { + "width": 399, + "height": 400 + }, + "flags": { + "strip_size": 4, + "reference_corner": "top_left", + "is_transposed": false, + "delta_s_offset": 3 + }, + "initial_strip_t": -4, + "strips": [ + ] + } + }, + + { + "segment_number": 3, + "type": "lossless_text_region", + "page_association": 1, + "data": { + "region_segment_information": { + "width": 399, + "height": 400 + }, + "flags": { + "strip_size": 4, + "reference_corner": "top_left", + "is_transposed": false, + "delta_s_offset": 3 + }, + "initial_strip_t": -4, + "strips": [ + ] + } + }, + + { + "segment_number": 4, + "type": "lossless_generic_region", + "page_association": 1, + "data": { + "image_data": { + "from_file": "bitmap.bmp" + } + } + }, + + { + "segment_number": 5, + "type": "end_of_page", + "page_association": 1 + } + ] +} diff --git a/Tests/LibGfx/test-inputs/jbig2/json/bitmap-symbol-negative-sbdsoffset.json b/Tests/LibGfx/test-inputs/jbig2/json/bitmap-symbol-negative-sbdsoffset.json index 38a1ddcaa8557c..408fa85c15cf0f 100644 --- a/Tests/LibGfx/test-inputs/jbig2/json/bitmap-symbol-negative-sbdsoffset.json +++ b/Tests/LibGfx/test-inputs/jbig2/json/bitmap-symbol-negative-sbdsoffset.json @@ -45,10 +45,21 @@ "image_data": { "from_file": "bitmap.bmp", "crop": { - "x": 232, - "y": 70, - "width": 120, - "height": 250 + "x": 50, + "y": 108, + "width": 30, + "height": 30 + } + } + }, + { + "image_data": { + "from_file": "bitmap.bmp", + "crop": { + "x": 60, + "y": 265, + "width": 30, + "height": 30 } } } @@ -75,21 +86,10 @@ "image_data": { "from_file": "bitmap.bmp", "crop": { - "x": 50, - "y": 108, - "width": 30, - "height": 30 - } - } - }, - { - "image_data": { - "from_file": "bitmap.bmp", - "crop": { - "x": 60, - "y": 265, - "width": 30, - "height": 30 + "x": 232, + "y": 70, + "width": 120, + "height": 250 } } } @@ -143,29 +143,44 @@ "initial_strip_t": -4, "strips": [ { - "strip_t": 4, + "strip_t": 68, "instances": [ { - "instance_s": 50, - "instance_t": 108, - "symbol_id": 2 - }, - { - "instance_s": 60, - "instance_t": 265, + "instance_s": 232, + "instance_t": 70, "symbol_id": 3 - }, + } + ] + }, + { + "strip_t": 100, + "instances": [ { "instance_s": 135, "instance_t": 100, - "symbol_id": 1 - }, + "symbol_id": 2 + } + ] + }, + { + "strip_t": 108, + "instances": [ { - "instance_s": 232, - "instance_t": 70, + "instance_s": 50, + "instance_t": 108, "symbol_id": 0 } ] + }, + { + "strip_t": 264, + "instances": [ + { + "instance_s": 60, + "instance_t": 265, + "symbol_id": 1 + } + ] } ], diff --git a/Tests/LibGfx/test-inputs/jbig2/json/bitmap-symbol-refine.json b/Tests/LibGfx/test-inputs/jbig2/json/bitmap-symbol-refine.json new file mode 100644 index 00000000000000..e22c82c6ecd150 --- /dev/null +++ b/Tests/LibGfx/test-inputs/jbig2/json/bitmap-symbol-refine.json @@ -0,0 +1,184 @@ +{ + "global_header": { + "organization": "sequential", + "number_of_pages": 1 + }, + "segments": [ + { + "segment_number": 0, + "type": "page_information", + "page_association": 1, + "data": { + "page_width": 399, + "page_height": 400, + "flags": { + "is_eventually_lossless": true + } + } + }, + + { + "segment_number": 1, + "type": "symbol_dictionary", + "page_association": 1, + "retained": true, + "data": { + "height_classes": [ + { + "symbols": [ + { + "image_data": { + "from_file": "bitmap.bmp", + "crop": { + "x": 50, + "y": 108, + "width": 30, + "height": 30 + } + } + } + ] + }, + { + "symbols": [ + { + "image_data": { + "from_file": "bitmap-blemish.bmp", + "crop": { + "x": 60, + "y": 265, + "width": 40, + "height": 75 + } + } + } + ] + }, + { + "symbols": [ + { + "image_data": { + "from_file": "bitmap.bmp", + "crop": { + "x": 135, + "y": 100, + "width": 100, + "height": 100 + } + } + } + ] + }, + { + "symbols": [ + { + "image_data": { + "from_file": "bitmap.bmp", + "crop": { + "x": 232, + "y": 70, + "width": 120, + "height": 250 + } + } + } + ] + } + ] + } + }, + + { + "segment_number": 2, + "type": "intermediate_text_region", + "page_association": 1, + "retained": true, + "referred_to_segments": [ + { + "segment_number": 1, + "retained": false + } + ], + "data": { + "region_segment_information": { + "width": 399, + "height": 400 + }, + "flags": { + "uses_huffman_encoding": false, + "uses_refinement_coding": true, + "strip_size": 2, + "reference_corner": "top_left", + "refinement_template": 0 + }, + + "initial_strip_t": -4, + "strips": [ + { + "strip_t": 70, + "instances": [ + { + "instance_s": 232, + "instance_t": 70, + "symbol_id": 3 + } + ] + }, + { + "strip_t": 100, + "instances": [ + { + "instance_s": 135, + "instance_t": 100, + "symbol_id": 2 + } + ] + }, + { + "strip_t": 108, + "instances": [ + { + "instance_s": 50, + "instance_t": 108, + "symbol_id": 0 + } + ] + }, + { + "strip_t": 264, + "instances": [ + { + "instance_s": 60, + "instance_t": 265, + "symbol_id": 1 + } + ] + } + ] + } + }, + + { + "segment_number": 3, + "type": "generic_refinement_region", + "page_association": 1, + "referred_to_segments": [ + { + "segment_number": 2, + "retained": false + } + ], + "data": { + "image_data": { + "from_file": "bitmap.bmp" + } + } + }, + + { + "segment_number": 4, + "type": "end_of_page", + "page_association": 1 + } + ] +} diff --git a/Tests/LibGfx/test-inputs/jbig2/json/bitmap-symbol-symbolrefine-textrefine.json b/Tests/LibGfx/test-inputs/jbig2/json/bitmap-symbol-symbolrefine-textrefine.json new file mode 100644 index 00000000000000..aa42e06fb8889e --- /dev/null +++ b/Tests/LibGfx/test-inputs/jbig2/json/bitmap-symbol-symbolrefine-textrefine.json @@ -0,0 +1,369 @@ +{ + "global_header": { + "organization": "sequential", + "number_of_pages": 1 + }, + "segments": [ + { + "segment_number": 0, + "type": "page_information", + "page_association": 1, + "data": { + "page_width": 399, + "page_height": 400, + "flags": { + "is_eventually_lossless": true + } + } + }, + + { + "segment_number": 1, + "type": "symbol_dictionary", + "page_association": 1, + "retained": true, + "data": { + "flags": { + "uses_huffman_encoding": false, + "uses_refinement_or_aggregate_coding": false, + "huffman_table_selection_for_height_differences": 0, + "huffman_table_selection_for_width_differences": 0, + "huffman_table_selection_for_bitmap_sizes": 0, + "huffman_table_selection_for_number_of_symbol_instances": 0, + "is_bitmap_coding_context_used": false, + "is_bitmap_coding_context_retained": false, + "template": 0, + "refinement_template": 0 + }, + + "height_classes": [ + { + "symbols": [ + { + "image_data": { + "from_file": "bitmap.bmp", + "crop": { + "x": 60, + "y": 265, + "width": 30, + "height": 30 + } + } + }, + { + "image_data": { + "from_file": "bitmap.bmp", + "crop": { + "x": 50, + "y": 108, + "width": 30, + "height": 30 + } + } + } + ] + }, + { + "symbols": [ + { + "image_data": { + "from_file": "bitmap.bmp", + "crop": { + "x": 50, + "y": 108, + "width": 40, + "height": 40 + } + } + }, + { + "image_data": { + "from_file": "bitmap.bmp", + "crop": { + "x": 135, + "y": 160, + "width": 60, + "height": 40 + } + } + } + ] + }, + { + "symbols": [ + { + "image_data": { + "from_file": "bitmap.bmp", + "crop": { + "x": 195, + "y": 100, + "width": 40, + "height": 60 + } + } + }, + { + "image_data": { + "from_file": "bitmap.bmp", + "crop": { + "x": 0, + "y": 0, + "width": 59, + "height": 60 + } + } + } + ] + }, + { + "symbols": [ + { + "image_data": { + "from_file": "bitmap.bmp", + "crop": { + "x": 232, + "y": 70, + "width": 120, + "height": 250 + } + } + } + ] + } + ] + } + }, + + { + "segment_number": 2, + "type": "symbol_dictionary", + "page_association": 1, + "retained": true, + "referred_to_segments": [ + { + "segment_number": 1, + "retained": false + } + ], + "data": { + "flags": { + "uses_huffman_encoding": false, + "uses_refinement_or_aggregate_coding": true, + "huffman_table_selection_for_height_differences": 0, + "huffman_table_selection_for_width_differences": 0, + "huffman_table_selection_for_bitmap_sizes": 0, + "huffman_table_selection_for_number_of_symbol_instances": 0, + "is_bitmap_coding_context_used": false, + "is_bitmap_coding_context_retained": false, + "template": 0, + "refinement_template": 0 + }, + + "height_classes": [ + { + "symbols": [ + { + "refines_symbol_to": { + "symbol_id": 5, + "delta_x_offset": 0, + "delta_y_offset": 0, + "image_data": { + "from_file": "bitmap.bmp", + "crop": { + "x": 146, + "y": 100, + "width": 59, + "height": 60 + } + } + } + }, + { + "width": 99, + "height": 60, + "refines_using_strips": { + "initial_strip_t": -1, + "strips": [ + { + "strip_t": 0, + "instances": [ + { + "instance_s": 0, + "instance_t": 0, + "symbol_id": 7 + }, + { + "instance_s": 60, + "instance_t": 0, + "symbol_id": 4 + } + ] + } + ] + } + }, + { + "refines_symbol_to": { + "symbol_id": 8, + "delta_x_offset": 0, + "delta_y_offset": 0, + "image_data": { + "from_file": "bitmap.bmp", + "crop": { + "x": 136, + "y": 100, + "width": 99, + "height": 60 + } + } + } + } + ] + }, + { + "symbols": [ + { + "width": 100, + "height": 100, + "refines_using_strips": { + "initial_strip_t": -1, + "strips": [ + { + "strip_t": 0, + "instances": [ + { + "instance_s": 1, + "instance_t": 0, + "symbol_id": 9 + } + ] + }, + { + "strip_t": 60, + "instances": [ + { + "instance_s": 0, + "instance_t": 60, + "symbol_id": 3 + }, + { + "instance_s": 60, + "instance_t": 60, + "symbol_id": 2 + } + ] + } + ] + } + } + ] + } + ] + } + }, + + { + "segment_number": 3, + "type": "text_region", + "page_association": 1, + "referred_to_segments": [ + { + "segment_number": 2, + "retained": false + } + ], + "data": { + "region_segment_information": { + "width": 399, + "height": 400 + }, + "flags": { + "uses_huffman_encoding": false, + "uses_refinement_coding": true, + "strip_size": 2, + "reference_corner": "top_left", + "is_transposed": false, + "combination_operator": "xor", + "default_pixel_value": "white", + "delta_s_offset": 3, + "refinement_template": 0 + }, + "huffman_flags": { + "huffman_table_selection_for_first_s": 0, + "huffman_table_selection_for_subsequent_s": 0, + "huffman_table_selection_for_t": 0, + "huffman_table_selection_for_refinement_delta_width": 0, + "huffman_table_selection_for_refinement_delta_height": 0, + "huffman_table_selection_for_refinement_delta_x_offset": 0, + "huffman_table_selection_for_refinement_delta_y_offset": 0, + "huffman_table_selection_for_refinement_size_table": 0 + }, + + "initial_strip_t": -4, + "strips": [ + { + "strip_t": 70, + "instances": [ + { + "instance_s": 232, + "instance_t": 70, + "symbol_id": 6 + } + ] + }, + { + "strip_t": 100, + "instances": [ + { + "instance_s": 135, + "instance_t": 100, + "symbol_id": 10, + "instance_refines_symbol_to": { + "delta_width": 0, + "delta_height": 0, + "delta_x_offset": 0, + "delta_y_offset": 0, + "image_data": { + "from_file": "bitmap.bmp", + "crop": { + "x": 135, + "y": 100, + "width": 100, + "height": 100 + } + } + } + } + ] + }, + { + "strip_t": 108, + "instances": [ + { + "instance_s": 50, + "instance_t": 108, + "symbol_id": 1 + } + ] + }, + { + "strip_t": 264, + "instances": [ + { + "instance_s": 60, + "instance_t": 265, + "symbol_id": 0 + } + ] + } + ] + } + }, + + { + "segment_number": 4, + "type": "end_of_page", + "page_association": 1 + } + ] +} diff --git a/Tests/LibGfx/test-inputs/jbig2/json/bitmap-symbol-symhuffrefine-textrefine.json b/Tests/LibGfx/test-inputs/jbig2/json/bitmap-symbol-symhuffrefine-textrefine.json new file mode 100644 index 00000000000000..b408251e877a9f --- /dev/null +++ b/Tests/LibGfx/test-inputs/jbig2/json/bitmap-symbol-symhuffrefine-textrefine.json @@ -0,0 +1,369 @@ +{ + "global_header": { + "organization": "sequential", + "number_of_pages": 1 + }, + "segments": [ + { + "segment_number": 0, + "type": "page_information", + "page_association": 1, + "data": { + "page_width": 399, + "page_height": 400, + "flags": { + "is_eventually_lossless": true + } + } + }, + + { + "segment_number": 1, + "type": "symbol_dictionary", + "page_association": 1, + "retained": true, + "data": { + "flags": { + "uses_huffman_encoding": true, + "uses_refinement_or_aggregate_coding": false, + "huffman_table_selection_for_height_differences": 0, + "huffman_table_selection_for_width_differences": 0, + "huffman_table_selection_for_bitmap_sizes": 0, + "huffman_table_selection_for_number_of_symbol_instances": 0, + "is_bitmap_coding_context_used": false, + "is_bitmap_coding_context_retained": false, + "template": 0, + "refinement_template": 0 + }, + + "height_classes": [ + { + "symbols": [ + { + "image_data": { + "from_file": "bitmap.bmp", + "crop": { + "x": 60, + "y": 265, + "width": 30, + "height": 30 + } + } + }, + { + "image_data": { + "from_file": "bitmap.bmp", + "crop": { + "x": 50, + "y": 108, + "width": 30, + "height": 30 + } + } + } + ] + }, + { + "symbols": [ + { + "image_data": { + "from_file": "bitmap.bmp", + "crop": { + "x": 50, + "y": 108, + "width": 40, + "height": 40 + } + } + }, + { + "image_data": { + "from_file": "bitmap.bmp", + "crop": { + "x": 135, + "y": 160, + "width": 60, + "height": 40 + } + } + } + ] + }, + { + "symbols": [ + { + "image_data": { + "from_file": "bitmap.bmp", + "crop": { + "x": 195, + "y": 100, + "width": 40, + "height": 60 + } + } + }, + { + "image_data": { + "from_file": "bitmap.bmp", + "crop": { + "x": 0, + "y": 0, + "width": 59, + "height": 60 + } + } + } + ] + }, + { + "symbols": [ + { + "image_data": { + "from_file": "bitmap.bmp", + "crop": { + "x": 232, + "y": 70, + "width": 120, + "height": 250 + } + } + } + ] + } + ] + } + }, + + { + "segment_number": 2, + "type": "symbol_dictionary", + "page_association": 1, + "retained": true, + "referred_to_segments": [ + { + "segment_number": 1, + "retained": false + } + ], + "data": { + "flags": { + "uses_huffman_encoding": true, + "uses_refinement_or_aggregate_coding": true, + "huffman_table_selection_for_height_differences": 0, + "huffman_table_selection_for_width_differences": 0, + "huffman_table_selection_for_bitmap_sizes": 0, + "huffman_table_selection_for_number_of_symbol_instances": 0, + "is_bitmap_coding_context_used": false, + "is_bitmap_coding_context_retained": false, + "template": 0, + "refinement_template": 0 + }, + + "height_classes": [ + { + "symbols": [ + { + "refines_symbol_to": { + "symbol_id": 5, + "delta_x_offset": 0, + "delta_y_offset": 0, + "image_data": { + "from_file": "bitmap.bmp", + "crop": { + "x": 146, + "y": 100, + "width": 59, + "height": 60 + } + } + } + }, + { + "width": 99, + "height": 60, + "refines_using_strips": { + "initial_strip_t": -1, + "strips": [ + { + "strip_t": 0, + "instances": [ + { + "instance_s": 0, + "instance_t": 0, + "symbol_id": 7 + }, + { + "instance_s": 60, + "instance_t": 0, + "symbol_id": 4 + } + ] + } + ] + } + }, + { + "refines_symbol_to": { + "symbol_id": 8, + "delta_x_offset": 0, + "delta_y_offset": 0, + "image_data": { + "from_file": "bitmap.bmp", + "crop": { + "x": 136, + "y": 100, + "width": 99, + "height": 60 + } + } + } + } + ] + }, + { + "symbols": [ + { + "width": 100, + "height": 100, + "refines_using_strips": { + "initial_strip_t": -1, + "strips": [ + { + "strip_t": 0, + "instances": [ + { + "instance_s": 1, + "instance_t": 0, + "symbol_id": 9 + } + ] + }, + { + "strip_t": 60, + "instances": [ + { + "instance_s": 0, + "instance_t": 60, + "symbol_id": 3 + }, + { + "instance_s": 60, + "instance_t": 60, + "symbol_id": 2 + } + ] + } + ] + } + } + ] + } + ] + } + }, + + { + "segment_number": 3, + "type": "text_region", + "page_association": 1, + "referred_to_segments": [ + { + "segment_number": 2, + "retained": false + } + ], + "data": { + "region_segment_information": { + "width": 399, + "height": 400 + }, + "flags": { + "uses_huffman_encoding": true, + "uses_refinement_coding": true, + "strip_size": 2, + "reference_corner": "top_left", + "is_transposed": false, + "combination_operator": "xor", + "default_pixel_value": "white", + "delta_s_offset": 3, + "refinement_template": 0 + }, + "huffman_flags": { + "huffman_table_selection_for_first_s": 0, + "huffman_table_selection_for_subsequent_s": 0, + "huffman_table_selection_for_t": 0, + "huffman_table_selection_for_refinement_delta_width": 0, + "huffman_table_selection_for_refinement_delta_height": 0, + "huffman_table_selection_for_refinement_delta_x_offset": 0, + "huffman_table_selection_for_refinement_delta_y_offset": 0, + "huffman_table_selection_for_refinement_size_table": 0 + }, + + "initial_strip_t": -4, + "strips": [ + { + "strip_t": 70, + "instances": [ + { + "instance_s": 232, + "instance_t": 70, + "symbol_id": 6 + } + ] + }, + { + "strip_t": 100, + "instances": [ + { + "instance_s": 135, + "instance_t": 100, + "symbol_id": 10, + "instance_refines_symbol_to": { + "delta_width": 0, + "delta_height": 0, + "delta_x_offset": 0, + "delta_y_offset": 0, + "image_data": { + "from_file": "bitmap.bmp", + "crop": { + "x": 135, + "y": 100, + "width": 100, + "height": 100 + } + } + } + } + ] + }, + { + "strip_t": 108, + "instances": [ + { + "instance_s": 50, + "instance_t": 108, + "symbol_id": 1 + } + ] + }, + { + "strip_t": 264, + "instances": [ + { + "instance_s": 60, + "instance_t": 265, + "symbol_id": 0 + } + ] + } + ] + } + }, + + { + "segment_number": 4, + "type": "end_of_page", + "page_association": 1 + } + ] +} diff --git a/Tests/LibGfx/test-inputs/jbig2/json/bitmap-symbol-textbottomleft.json b/Tests/LibGfx/test-inputs/jbig2/json/bitmap-symbol-textbottomleft.json index 7f0e2d6f8cdc4b..2d2c08594bd467 100644 --- a/Tests/LibGfx/test-inputs/jbig2/json/bitmap-symbol-textbottomleft.json +++ b/Tests/LibGfx/test-inputs/jbig2/json/bitmap-symbol-textbottomleft.json @@ -30,10 +30,21 @@ "image_data": { "from_file": "bitmap.bmp", "crop": { - "x": 232, - "y": 70, - "width": 120, - "height": 250 + "x": 50, + "y": 108, + "width": 30, + "height": 30 + } + } + }, + { + "image_data": { + "from_file": "bitmap.bmp", + "crop": { + "x": 60, + "y": 265, + "width": 30, + "height": 30 } } } @@ -46,24 +57,20 @@ "from_file": "bitmap.bmp", "crop": { "x": 135, - "y": 100, - "width": 100, + "y": 101, + "width": 25, "height": 100 } } - } - ] - }, - { - "symbols": [ + }, { "image_data": { "from_file": "bitmap.bmp", "crop": { - "x": 50, - "y": 108, - "width": 30, - "height": 30 + "x": 160, + "y": 102, + "width": 25, + "height": 100 } } }, @@ -71,10 +78,36 @@ "image_data": { "from_file": "bitmap.bmp", "crop": { - "x": 60, - "y": 265, - "width": 30, - "height": 30 + "x": 185, + "y": 103, + "width": 25, + "height": 100 + } + } + }, + { + "image_data": { + "from_file": "bitmap.bmp", + "crop": { + "x": 210, + "y": 102, + "width": 25, + "height": 100 + } + } + } + ] + }, + { + "symbols": [ + { + "image_data": { + "from_file": "bitmap.bmp", + "crop": { + "x": 232, + "y": 70, + "width": 120, + "height": 250 } } } @@ -108,27 +141,57 @@ "initial_strip_t": -4, "strips": [ { - "strip_t": 4, + "strip_t": 136, "instances": [ { "instance_s": 50, "instance_t": 137, + "symbol_id": 0 + } + ] + }, + { + "strip_t": 200, + "instances": [ + { + "instance_s": 135, + "instance_t": 200, "symbol_id": 2 }, { - "instance_s": 60, - "instance_t": 294, + "instance_s": 160, + "instance_t": 201, "symbol_id": 3 }, { - "instance_s": 135, - "instance_t": 199, - "symbol_id": 1 + "instance_s": 185, + "instance_t": 202, + "symbol_id": 4 }, + { + "instance_s": 210, + "instance_t": 201, + "symbol_id": 5 + } + ] + }, + { + "strip_t": 292, + "instances": [ + { + "instance_s": 60, + "instance_t": 294, + "symbol_id": 1 + } + ] + }, + { + "strip_t": 316, + "instances": [ { "instance_s": 232, "instance_t": 319, - "symbol_id": 0 + "symbol_id": 6 } ] } diff --git a/Tests/LibGfx/test-inputs/jbig2/json/bitmap-symbol-textbottomlefttranspose.json b/Tests/LibGfx/test-inputs/jbig2/json/bitmap-symbol-textbottomlefttranspose.json new file mode 100644 index 00000000000000..01e41179e29b9f --- /dev/null +++ b/Tests/LibGfx/test-inputs/jbig2/json/bitmap-symbol-textbottomlefttranspose.json @@ -0,0 +1,209 @@ +{ + "global_header": { + "organization": "sequential", + "number_of_pages": 1 + }, + "segments": [ + { + "segment_number": 0, + "type": "page_information", + "page_association": 1, + "data": { + "page_width": 399, + "page_height": 400, + "flags": { + "is_eventually_lossless": true + } + } + }, + + { + "segment_number": 1, + "type": "symbol_dictionary", + "page_association": 1, + "retained": true, + "data": { + "height_classes": [ + { + "symbols": [ + { + "image_data": { + "from_file": "bitmap.bmp", + "crop": { + "x": 135, + "y": 100, + "width": 100, + "height": 25 + } + } + }, + { + "image_data": { + "from_file": "bitmap.bmp", + "crop": { + "x": 134, + "y": 125, + "width": 100, + "height": 25 + } + } + }, + { + "image_data": { + "from_file": "bitmap.bmp", + "crop": { + "x": 133, + "y": 150, + "width": 100, + "height": 25 + } + } + }, + { + "image_data": { + "from_file": "bitmap.bmp", + "crop": { + "x": 134, + "y": 175, + "width": 100, + "height": 25 + } + } + } + ] + }, + { + "symbols": [ + { + "image_data": { + "from_file": "bitmap.bmp", + "crop": { + "x": 50, + "y": 108, + "width": 30, + "height": 30 + } + } + }, + { + "image_data": { + "from_file": "bitmap.bmp", + "crop": { + "x": 60, + "y": 265, + "width": 30, + "height": 30 + } + } + } + ] + }, + { + "symbols": [ + { + "image_data": { + "from_file": "bitmap.bmp", + "crop": { + "x": 232, + "y": 70, + "width": 120, + "height": 250 + } + } + } + ] + } + ] + } + }, + + { + "segment_number": 2, + "type": "lossless_text_region", + "page_association": 1, + "referred_to_segments": [ + { + "segment_number": 1, + "retained": false + } + ], + "data": { + "region_segment_information": { + "width": 399, + "height": 400 + }, + "flags": { + "strip_size": 4, + "reference_corner": "bottom_left", + "is_transposed": true, + "delta_s_offset": 3 + }, + + "initial_strip_t": -4, + "strips": [ + { + "strip_t": 48, + "instances": [ + { + "instance_s": 108, + "instance_t": 50, + "symbol_id": 4 + } + ] + }, + { + "strip_t": 60, + "instances": [ + { + "instance_s": 265, + "instance_t": 60, + "symbol_id": 5 + } + ] + }, + { + "strip_t": 132, + "instances": [ + { + "instance_s": 100, + "instance_t": 135, + "symbol_id": 0 + }, + { + "instance_s": 125, + "instance_t": 134, + "symbol_id": 1 + }, + { + "instance_s": 150, + "instance_t": 133, + "symbol_id": 2 + }, + { + "instance_s": 175, + "instance_t": 134, + "symbol_id": 3 + } + ] + }, + { + "strip_t": 232, + "instances": [ + { + "instance_s": 70, + "instance_t": 232, + "symbol_id": 6 + } + ] + } + ] + } + }, + + { + "segment_number": 3, + "type": "end_of_page", + "page_association": 1 + } + ] +} diff --git a/Tests/LibGfx/test-inputs/jbig2/json/bitmap-symbol-textbottomright.json b/Tests/LibGfx/test-inputs/jbig2/json/bitmap-symbol-textbottomright.json index 067784ec0c4b53..114b89e3d7ac66 100644 --- a/Tests/LibGfx/test-inputs/jbig2/json/bitmap-symbol-textbottomright.json +++ b/Tests/LibGfx/test-inputs/jbig2/json/bitmap-symbol-textbottomright.json @@ -30,10 +30,21 @@ "image_data": { "from_file": "bitmap.bmp", "crop": { - "x": 232, - "y": 70, - "width": 120, - "height": 250 + "x": 50, + "y": 108, + "width": 30, + "height": 30 + } + } + }, + { + "image_data": { + "from_file": "bitmap.bmp", + "crop": { + "x": 60, + "y": 265, + "width": 30, + "height": 30 } } } @@ -46,24 +57,20 @@ "from_file": "bitmap.bmp", "crop": { "x": 135, - "y": 100, - "width": 100, + "y": 101, + "width": 25, "height": 100 } } - } - ] - }, - { - "symbols": [ + }, { "image_data": { "from_file": "bitmap.bmp", "crop": { - "x": 50, - "y": 108, - "width": 30, - "height": 30 + "x": 160, + "y": 102, + "width": 25, + "height": 100 } } }, @@ -71,10 +78,36 @@ "image_data": { "from_file": "bitmap.bmp", "crop": { - "x": 60, - "y": 265, - "width": 30, - "height": 30 + "x": 185, + "y": 103, + "width": 25, + "height": 100 + } + } + }, + { + "image_data": { + "from_file": "bitmap.bmp", + "crop": { + "x": 210, + "y": 102, + "width": 25, + "height": 100 + } + } + } + ] + }, + { + "symbols": [ + { + "image_data": { + "from_file": "bitmap.bmp", + "crop": { + "x": 232, + "y": 70, + "width": 120, + "height": 250 } } } @@ -108,27 +141,57 @@ "initial_strip_t": -4, "strips": [ { - "strip_t": 4, + "strip_t": 136, "instances": [ { "instance_s": 50, "instance_t": 137, + "symbol_id": 0 + } + ] + }, + { + "strip_t": 200, + "instances": [ + { + "instance_s": 135, + "instance_t": 200, "symbol_id": 2 }, { - "instance_s": 60, - "instance_t": 294, + "instance_s": 160, + "instance_t": 201, "symbol_id": 3 }, { - "instance_s": 135, - "instance_t": 199, - "symbol_id": 1 + "instance_s": 185, + "instance_t": 202, + "symbol_id": 4 }, + { + "instance_s": 210, + "instance_t": 201, + "symbol_id": 5 + } + ] + }, + { + "strip_t": 292, + "instances": [ + { + "instance_s": 60, + "instance_t": 294, + "symbol_id": 1 + } + ] + }, + { + "strip_t": 316, + "instances": [ { "instance_s": 232, "instance_t": 319, - "symbol_id": 0 + "symbol_id": 6 } ] } diff --git a/Tests/LibGfx/test-inputs/jbig2/json/bitmap-symbol-textbottomrighttranspose.json b/Tests/LibGfx/test-inputs/jbig2/json/bitmap-symbol-textbottomrighttranspose.json new file mode 100644 index 00000000000000..25fac1618aa355 --- /dev/null +++ b/Tests/LibGfx/test-inputs/jbig2/json/bitmap-symbol-textbottomrighttranspose.json @@ -0,0 +1,209 @@ +{ + "global_header": { + "organization": "sequential", + "number_of_pages": 1 + }, + "segments": [ + { + "segment_number": 0, + "type": "page_information", + "page_association": 1, + "data": { + "page_width": 399, + "page_height": 400, + "flags": { + "is_eventually_lossless": true + } + } + }, + + { + "segment_number": 1, + "type": "symbol_dictionary", + "page_association": 1, + "retained": true, + "data": { + "height_classes": [ + { + "symbols": [ + { + "image_data": { + "from_file": "bitmap.bmp", + "crop": { + "x": 135, + "y": 100, + "width": 100, + "height": 25 + } + } + }, + { + "image_data": { + "from_file": "bitmap.bmp", + "crop": { + "x": 134, + "y": 125, + "width": 100, + "height": 25 + } + } + }, + { + "image_data": { + "from_file": "bitmap.bmp", + "crop": { + "x": 133, + "y": 150, + "width": 100, + "height": 25 + } + } + }, + { + "image_data": { + "from_file": "bitmap.bmp", + "crop": { + "x": 134, + "y": 175, + "width": 100, + "height": 25 + } + } + } + ] + }, + { + "symbols": [ + { + "image_data": { + "from_file": "bitmap.bmp", + "crop": { + "x": 50, + "y": 108, + "width": 30, + "height": 30 + } + } + }, + { + "image_data": { + "from_file": "bitmap.bmp", + "crop": { + "x": 60, + "y": 265, + "width": 30, + "height": 30 + } + } + } + ] + }, + { + "symbols": [ + { + "image_data": { + "from_file": "bitmap.bmp", + "crop": { + "x": 232, + "y": 70, + "width": 120, + "height": 250 + } + } + } + ] + } + ] + } + }, + + { + "segment_number": 2, + "type": "lossless_text_region", + "page_association": 1, + "referred_to_segments": [ + { + "segment_number": 1, + "retained": false + } + ], + "data": { + "region_segment_information": { + "width": 399, + "height": 400 + }, + "flags": { + "strip_size": 4, + "reference_corner": "bottom_right", + "is_transposed": true, + "delta_s_offset": 3 + }, + + "initial_strip_t": -4, + "strips": [ + { + "strip_t": 76, + "instances": [ + { + "instance_s": 108, + "instance_t": 79, + "symbol_id": 4 + } + ] + }, + { + "strip_t": 88, + "instances": [ + { + "instance_s": 265, + "instance_t": 89, + "symbol_id": 5 + } + ] + }, + { + "strip_t": 232, + "instances": [ + { + "instance_s": 100, + "instance_t": 234, + "symbol_id": 0 + }, + { + "instance_s": 125, + "instance_t": 233, + "symbol_id": 1 + }, + { + "instance_s": 150, + "instance_t": 232, + "symbol_id": 2 + }, + { + "instance_s": 175, + "instance_t": 233, + "symbol_id": 3 + } + ] + }, + { + "strip_t": 348, + "instances": [ + { + "instance_s": 70, + "instance_t": 351, + "symbol_id": 6 + } + ] + } + ] + } + }, + + { + "segment_number": 3, + "type": "end_of_page", + "page_association": 1 + } + ] +} diff --git a/Tests/LibGfx/test-inputs/jbig2/json/bitmap-symbol-textcomposite.json b/Tests/LibGfx/test-inputs/jbig2/json/bitmap-symbol-textcomposite.json new file mode 100644 index 00000000000000..1e9d2cb8e3423c --- /dev/null +++ b/Tests/LibGfx/test-inputs/jbig2/json/bitmap-symbol-textcomposite.json @@ -0,0 +1,390 @@ +{ + "global_header": { + "organization": "sequential", + "number_of_pages": 1 + }, + "segments": [ + { + "segment_number": 0, + "type": "page_information", + "page_association": 1, + "data": { + "page_width": 399, + "page_height": 400, + "flags": { + "default_color": "white", + "is_eventually_lossless": true, + "default_combination_operator": "xor" + } + } + }, + + { + "segment_number": 1, + "type": "symbol_dictionary", + "page_association": 1, + "retained": true, + "data": { + "height_classes": [ + { + "symbols": [ + { + "image_data": { + "from_file": "bitmap.bmp", + "crop": { + "x": 60, + "y": 265, + "width": 30, + "height": 30 + }, + "invert": true + } + }, + { + "image_data": { + "from_file": "bitmap.bmp", + "crop": { + "x": 50, + "y": 108, + "width": 30, + "height": 30 + }, + "invert": true + } + }, + { + "image_data": { + "from_file": "bitmap.bmp", + "crop": { + "x": 0, + "y": 0, + "width": 30, + "height": 30 + }, + "invert": true + } + } + ] + }, + { + "symbols": [ + { + "image_data": { + "from_file": "bitmap.bmp", + "crop": { + "x": 135, + "y": 100, + "width": 100, + "height": 100 + } + } + } + ] + }, + { + "symbols": [ + { + "image_data": { + "from_file": "bitmap.bmp", + "crop": { + "x": 352, + "y": 70, + "width": 40, + "height": 250 + } + } + }, + { + "image_data": { + "from_file": "bitmap.bmp", + "crop": { + "x": 232, + "y": 70, + "width": 120, + "height": 250 + } + } + } + ] + } + ] + } + }, + + { + "segment_number": 2, + "type": "text_region", + "page_association": 1, + "referred_to_segments": [ + { + "segment_number": 1, + "retained": true + } + ], + "data": { + "region_segment_information": { + "x": 232, + "y": 70, + "width": 120, + "height": 250, + "flags": { + "external_combination_operator": "xor" + } + }, + "flags": { + "strip_size": 1, + "reference_corner": "top_left", + "combination_operator": "or", + "default_pixel_value": "white", + "delta_s_offset": 0 + }, + "initial_strip_t": -2, + "strips": [ + { + "strip_t": 0, + "instances": [ + { + "instance_s": 0, + "instance_t": 0, + "symbol_id": 5 + }, + { + "instance_s": 0, + "instance_t": 0, + "symbol_id": 5 + }, + { + "instance_s": 60, + "instance_t": 0, + "symbol_id": 4 + } + ] + } + ] + } + }, + + { + "segment_number": 3, + "type": "text_region", + "page_association": 1, + "referred_to_segments": [ + { + "segment_number": 1, + "retained": true + } + ], + "data": { + "region_segment_information": { + "x": 135, + "y": 100, + "width": 100, + "height": 100, + "flags": { + "external_combination_operator": "xor" + } + }, + "flags": { + "strip_size": 1, + "reference_corner": "top_left", + "combination_operator": "xor", + "default_pixel_value": "white", + "delta_s_offset": 0 + }, + "initial_strip_t": -2, + "strips": [ + { + "strip_t": 0, + "instances": [ + { + "instance_s": 0, + "instance_t": 0, + "symbol_id": 3 + } + ] + }, + { + "strip_t": 20, + "instances": [ + { + "instance_s": 10, + "instance_t": 20, + "symbol_id": 3 + }, + { + "instance_s": 10, + "instance_t": 20, + "symbol_id": 3 + } + ] + } + ] + } + }, + + { + "segment_number": 5, + "type": "lossless_generic_region", + "page_association": 1, + "data": { + "region_segment_information": { + "x": 50, + "y": 108, + "flags": { + "external_combination_operator": "xor" + } + }, + "image_data": { + "from_file": "bitmap.bmp", + "crop": { + "x": 0, + "y": 0, + "width": 30, + "height": 30 + }, + "invert": true + } + } + }, + { + "segment_number": 6, + "type": "text_region", + "page_association": 1, + "referred_to_segments": [ + { + "segment_number": 1, + "retained": true + } + ], + "data": { + "region_segment_information": { + "x": 50, + "y": 108, + "width": 30, + "height": 30, + "flags": { + "external_combination_operator": "xor" + } + }, + "flags": { + "strip_size": 2, + "reference_corner": "top_left", + "combination_operator": "and", + "default_pixel_value": "black", + "delta_s_offset": 3 + }, + "initial_strip_t": -2, + "strips": [ + { + "strip_t": 0, + "instances": [ + { + "instance_s": 0, + "instance_t": 0, + "symbol_id": 1 + }, + { + "instance_s": 0, + "instance_t": 0, + "symbol_id": 1 + }, + { + "instance_s": 0, + "instance_t": 0, + "symbol_id": 2 + } + ] + } + ] + } + }, + + { + "segment_number": 7, + "type": "lossless_generic_region", + "page_association": 1, + "data": { + "region_segment_information": { + "x": 60, + "y": 265, + "flags": { + "external_combination_operator": "xor" + } + }, + "image_data": { + "from_file": "bitmap.bmp", + "crop": { + "x": 0, + "y": 0, + "width": 30, + "height": 30 + }, + "invert": true + } + } + }, + { + "segment_number": 8, + "type": "text_region", + "page_association": 1, + "referred_to_segments": [ + { + "segment_number": 1, + "retained": false + } + ], + "data": { + "region_segment_information": { + "x": 60, + "y": 265, + "width": 30, + "height": 30, + "flags": { + "external_combination_operator": "xor" + } + }, + "flags": { + "strip_size": 1, + "reference_corner": "top_left", + "combination_operator": "xnor", + "default_pixel_value": "black", + "delta_s_offset": 0 + }, + "initial_strip_t": -2, + "strips": [ + { + "strip_t": 0, + "instances": [ + { + "instance_s": 0, + "instance_t": 0, + "symbol_id": 0 + } + ] + }, + { + "strip_t": 20, + "instances": [ + { + "instance_s": 10, + "instance_t": 20, + "symbol_id": 3 + }, + { + "instance_s": 10, + "instance_t": 20, + "symbol_id": 3 + } + ] + } + ] + } + }, + + { + "segment_number": 9, + "type": "end_of_page", + "page_association": 1 + } + ] +} diff --git a/Tests/LibGfx/test-inputs/jbig2/json/bitmap-symbol-texttopright.json b/Tests/LibGfx/test-inputs/jbig2/json/bitmap-symbol-texttopright.json index 7278287fe8999d..80e100435b123d 100644 --- a/Tests/LibGfx/test-inputs/jbig2/json/bitmap-symbol-texttopright.json +++ b/Tests/LibGfx/test-inputs/jbig2/json/bitmap-symbol-texttopright.json @@ -30,10 +30,21 @@ "image_data": { "from_file": "bitmap.bmp", "crop": { - "x": 232, - "y": 70, - "width": 120, - "height": 250 + "x": 50, + "y": 108, + "width": 30, + "height": 30 + } + } + }, + { + "image_data": { + "from_file": "bitmap.bmp", + "crop": { + "x": 60, + "y": 265, + "width": 30, + "height": 30 } } } @@ -47,23 +58,19 @@ "crop": { "x": 135, "y": 100, - "width": 100, + "width": 25, "height": 100 } } - } - ] - }, - { - "symbols": [ + }, { "image_data": { "from_file": "bitmap.bmp", "crop": { - "x": 50, - "y": 108, - "width": 30, - "height": 30 + "x": 160, + "y": 101, + "width": 25, + "height": 100 } } }, @@ -71,10 +78,36 @@ "image_data": { "from_file": "bitmap.bmp", "crop": { - "x": 60, - "y": 265, - "width": 30, - "height": 30 + "x": 185, + "y": 102, + "width": 25, + "height": 100 + } + } + }, + { + "image_data": { + "from_file": "bitmap.bmp", + "crop": { + "x": 210, + "y": 101, + "width": 25, + "height": 100 + } + } + } + ] + }, + { + "symbols": [ + { + "image_data": { + "from_file": "bitmap.bmp", + "crop": { + "x": 232, + "y": 70, + "width": 120, + "height": 250 } } } @@ -102,35 +135,66 @@ "flags": { "strip_size": 4, "reference_corner": "top_right", - "is_transposed": false + "is_transposed": false, + "delta_s_offset": 3 }, "initial_strip_t": -4, "strips": [ { - "strip_t": 4, + "strip_t": 68, "instances": [ { - "instance_s": 50, - "instance_t": 108, + "instance_s": 232, + "instance_t": 70, + "symbol_id": 6 + } + ] + }, + { + "strip_t": 100, + "instances": [ + { + "instance_s": 135, + "instance_t": 100, "symbol_id": 2 }, { - "instance_s": 60, - "instance_t": 265, + "instance_s": 160, + "instance_t": 101, "symbol_id": 3 }, { - "instance_s": 135, - "instance_t": 100, - "symbol_id": 1 + "instance_s": 185, + "instance_t": 102, + "symbol_id": 4 }, { - "instance_s": 232, - "instance_t": 70, + "instance_s": 210, + "instance_t": 101, + "symbol_id": 5 + } + ] + }, + { + "strip_t": 108, + "instances": [ + { + "instance_s": 50, + "instance_t": 108, "symbol_id": 0 } ] + }, + { + "strip_t": 264, + "instances": [ + { + "instance_s": 60, + "instance_t": 265, + "symbol_id": 1 + } + ] } ] } diff --git a/Tests/LibGfx/test-inputs/jbig2/json/bitmap-symbol-texttoprighttranspose.json b/Tests/LibGfx/test-inputs/jbig2/json/bitmap-symbol-texttoprighttranspose.json new file mode 100644 index 00000000000000..eccac2ca325749 --- /dev/null +++ b/Tests/LibGfx/test-inputs/jbig2/json/bitmap-symbol-texttoprighttranspose.json @@ -0,0 +1,209 @@ +{ + "global_header": { + "organization": "sequential", + "number_of_pages": 1 + }, + "segments": [ + { + "segment_number": 0, + "type": "page_information", + "page_association": 1, + "data": { + "page_width": 399, + "page_height": 400, + "flags": { + "is_eventually_lossless": true + } + } + }, + + { + "segment_number": 1, + "type": "symbol_dictionary", + "page_association": 1, + "retained": true, + "data": { + "height_classes": [ + { + "symbols": [ + { + "image_data": { + "from_file": "bitmap.bmp", + "crop": { + "x": 135, + "y": 100, + "width": 100, + "height": 25 + } + } + }, + { + "image_data": { + "from_file": "bitmap.bmp", + "crop": { + "x": 134, + "y": 125, + "width": 100, + "height": 25 + } + } + }, + { + "image_data": { + "from_file": "bitmap.bmp", + "crop": { + "x": 133, + "y": 150, + "width": 100, + "height": 25 + } + } + }, + { + "image_data": { + "from_file": "bitmap.bmp", + "crop": { + "x": 134, + "y": 175, + "width": 100, + "height": 25 + } + } + } + ] + }, + { + "symbols": [ + { + "image_data": { + "from_file": "bitmap.bmp", + "crop": { + "x": 50, + "y": 108, + "width": 30, + "height": 30 + } + } + }, + { + "image_data": { + "from_file": "bitmap.bmp", + "crop": { + "x": 60, + "y": 265, + "width": 30, + "height": 30 + } + } + } + ] + }, + { + "symbols": [ + { + "image_data": { + "from_file": "bitmap.bmp", + "crop": { + "x": 232, + "y": 70, + "width": 120, + "height": 250 + } + } + } + ] + } + ] + } + }, + + { + "segment_number": 2, + "type": "lossless_text_region", + "page_association": 1, + "referred_to_segments": [ + { + "segment_number": 1, + "retained": false + } + ], + "data": { + "region_segment_information": { + "width": 399, + "height": 400 + }, + "flags": { + "strip_size": 4, + "reference_corner": "top_right", + "is_transposed": true, + "delta_s_offset": 3 + }, + + "initial_strip_t": -4, + "strips": [ + { + "strip_t": 76, + "instances": [ + { + "instance_s": 108, + "instance_t": 79, + "symbol_id": 4 + } + ] + }, + { + "strip_t": 88, + "instances": [ + { + "instance_s": 265, + "instance_t": 89, + "symbol_id": 5 + } + ] + }, + { + "strip_t": 232, + "instances": [ + { + "instance_s": 100, + "instance_t": 234, + "symbol_id": 0 + }, + { + "instance_s": 125, + "instance_t": 233, + "symbol_id": 1 + }, + { + "instance_s": 150, + "instance_t": 232, + "symbol_id": 2 + }, + { + "instance_s": 175, + "instance_t": 233, + "symbol_id": 3 + } + ] + }, + { + "strip_t": 348, + "instances": [ + { + "instance_s": 70, + "instance_t": 351, + "symbol_id": 6 + } + ] + } + ] + } + }, + + { + "segment_number": 3, + "type": "end_of_page", + "page_association": 1 + } + ] +} diff --git a/Tests/LibGfx/test-inputs/jbig2/json/bitmap-symbol-texttranspose.json b/Tests/LibGfx/test-inputs/jbig2/json/bitmap-symbol-texttranspose.json new file mode 100644 index 00000000000000..3138b7d6ffb24d --- /dev/null +++ b/Tests/LibGfx/test-inputs/jbig2/json/bitmap-symbol-texttranspose.json @@ -0,0 +1,209 @@ +{ + "global_header": { + "organization": "sequential", + "number_of_pages": 1 + }, + "segments": [ + { + "segment_number": 0, + "type": "page_information", + "page_association": 1, + "data": { + "page_width": 399, + "page_height": 400, + "flags": { + "is_eventually_lossless": true + } + } + }, + + { + "segment_number": 1, + "type": "symbol_dictionary", + "page_association": 1, + "retained": true, + "data": { + "height_classes": [ + { + "symbols": [ + { + "image_data": { + "from_file": "bitmap.bmp", + "crop": { + "x": 135, + "y": 100, + "width": 100, + "height": 25 + } + } + }, + { + "image_data": { + "from_file": "bitmap.bmp", + "crop": { + "x": 134, + "y": 125, + "width": 100, + "height": 25 + } + } + }, + { + "image_data": { + "from_file": "bitmap.bmp", + "crop": { + "x": 133, + "y": 150, + "width": 100, + "height": 25 + } + } + }, + { + "image_data": { + "from_file": "bitmap.bmp", + "crop": { + "x": 134, + "y": 175, + "width": 100, + "height": 25 + } + } + } + ] + }, + { + "symbols": [ + { + "image_data": { + "from_file": "bitmap.bmp", + "crop": { + "x": 50, + "y": 108, + "width": 30, + "height": 30 + } + } + }, + { + "image_data": { + "from_file": "bitmap.bmp", + "crop": { + "x": 60, + "y": 265, + "width": 30, + "height": 30 + } + } + } + ] + }, + { + "symbols": [ + { + "image_data": { + "from_file": "bitmap.bmp", + "crop": { + "x": 232, + "y": 70, + "width": 120, + "height": 250 + } + } + } + ] + } + ] + } + }, + + { + "segment_number": 2, + "type": "lossless_text_region", + "page_association": 1, + "referred_to_segments": [ + { + "segment_number": 1, + "retained": false + } + ], + "data": { + "region_segment_information": { + "width": 399, + "height": 400 + }, + "flags": { + "strip_size": 4, + "reference_corner": "top_left", + "is_transposed": true, + "delta_s_offset": 3 + }, + + "initial_strip_t": -4, + "strips": [ + { + "strip_t": 48, + "instances": [ + { + "instance_s": 108, + "instance_t": 50, + "symbol_id": 4 + } + ] + }, + { + "strip_t": 60, + "instances": [ + { + "instance_s": 265, + "instance_t": 60, + "symbol_id": 5 + } + ] + }, + { + "strip_t": 132, + "instances": [ + { + "instance_s": 100, + "instance_t": 135, + "symbol_id": 0 + }, + { + "instance_s": 125, + "instance_t": 134, + "symbol_id": 1 + }, + { + "instance_s": 150, + "instance_t": 133, + "symbol_id": 2 + }, + { + "instance_s": 175, + "instance_t": 134, + "symbol_id": 3 + } + ] + }, + { + "strip_t": 232, + "instances": [ + { + "instance_s": 70, + "instance_t": 232, + "symbol_id": 6 + } + ] + } + ] + } + }, + + { + "segment_number": 3, + "type": "end_of_page", + "page_association": 1 + } + ] +} diff --git a/Tests/LibGfx/test-inputs/jbig2/json/bitmap-symbol.json b/Tests/LibGfx/test-inputs/jbig2/json/bitmap-symbol.json index c2f29822492bff..1482f69b6ecc00 100644 --- a/Tests/LibGfx/test-inputs/jbig2/json/bitmap-symbol.json +++ b/Tests/LibGfx/test-inputs/jbig2/json/bitmap-symbol.json @@ -23,21 +23,6 @@ "page_association": 1, "retained": true, "data": { - "flags": { - "uses_huffman_encoding": false, - "uses_refinement_or_aggregate_coding": false, - "huffman_table_selection_for_height_differences": 0, - "huffman_table_selection_for_width_differences": 0, - "huffman_table_selection_for_bitmap_sizes": 0, - "huffman_table_selection_for_number_of_symbol_instances": 0, - "is_bitmap_coding_context_used": false, - "is_bitmap_coding_context_retained": false, - "template": 0, - "refinement_template": 0 - }, - "adaptive_template_pixels": [], - "refinement_adaptive_template_pixels": [], - "height_classes": [ { "symbols": [ @@ -45,10 +30,21 @@ "image_data": { "from_file": "bitmap.bmp", "crop": { - "x": 232, - "y": 70, - "width": 120, - "height": 250 + "x": 50, + "y": 108, + "width": 30, + "height": 30 + } + } + }, + { + "image_data": { + "from_file": "bitmap.bmp", + "crop": { + "x": 60, + "y": 265, + "width": 30, + "height": 30 } } } @@ -62,23 +58,19 @@ "crop": { "x": 135, "y": 100, - "width": 100, + "width": 25, "height": 100 } } - } - ] - }, - { - "symbols": [ + }, { "image_data": { "from_file": "bitmap.bmp", "crop": { - "x": 50, - "y": 108, - "width": 30, - "height": 30 + "x": 160, + "y": 101, + "width": 25, + "height": 100 } } }, @@ -86,18 +78,42 @@ "image_data": { "from_file": "bitmap.bmp", "crop": { - "x": 60, - "y": 265, - "width": 30, - "height": 30 + "x": 185, + "y": 102, + "width": 25, + "height": 100 + } + } + }, + { + "image_data": { + "from_file": "bitmap.bmp", + "crop": { + "x": 210, + "y": 101, + "width": 25, + "height": 100 + } + } + } + ] + }, + { + "symbols": [ + { + "image_data": { + "from_file": "bitmap.bmp", + "crop": { + "x": 232, + "y": 70, + "width": 120, + "height": 250 } } } ] } - ], - - "strip_trailing_7fffs": false + ] } }, @@ -117,59 +133,70 @@ "height": 400 }, "flags": { - "uses_huffman_encoding": false, - "uses_refinement_coding": false, "strip_size": 4, "reference_corner": "top_left", "is_transposed": false, - "combination_operator": "or", - "default_pixel_value": "white", - "delta_s_offset": 3, - "refinement_template": 0 - }, - "huffman_flags": { - "huffman_table_selection_for_first_s": 0, - "huffman_table_selection_for_subsequent_s": 0, - "huffman_table_selection_for_t": 0, - "huffman_table_selection_for_refinement_delta_width": 0, - "huffman_table_selection_for_refinement_delta_height": 0, - "huffman_table_selection_for_refinement_delta_x_offset": 0, - "huffman_table_selection_for_refinement_delta_y_offset": 0, - "huffman_table_selection_for_refinement_size_table": 0 + "delta_s_offset": 3 }, - "refinement_adaptive_template_pixels": [ - ], "initial_strip_t": -4, "strips": [ { - "strip_t": 4, + "strip_t": 68, "instances": [ { - "instance_s": 50, - "instance_t": 108, + "instance_s": 232, + "instance_t": 70, + "symbol_id": 6 + } + ] + }, + { + "strip_t": 100, + "instances": [ + { + "instance_s": 135, + "instance_t": 100, "symbol_id": 2 }, { - "instance_s": 60, - "instance_t": 265, + "instance_s": 160, + "instance_t": 101, "symbol_id": 3 }, { - "instance_s": 135, - "instance_t": 100, - "symbol_id": 1 + "instance_s": 185, + "instance_t": 102, + "symbol_id": 4 }, { - "instance_s": 232, - "instance_t": 70, + "instance_s": 210, + "instance_t": 101, + "symbol_id": 5 + } + ] + }, + { + "strip_t": 108, + "instances": [ + { + "instance_s": 50, + "instance_t": 108, "symbol_id": 0 } ] + }, + { + "strip_t": 264, + "instances": [ + { + "instance_s": 60, + "instance_t": 265, + "symbol_id": 1 + } + ] } - ], - - "strip_trailing_7fffs": false + ] } }, diff --git a/Tests/LibGfx/test-inputs/jbig2/json/bitmap-template1-tpgdon.json b/Tests/LibGfx/test-inputs/jbig2/json/bitmap-template1-tpgdon.json index e59ebbfd4c7ac4..27559a61e20dd2 100644 --- a/Tests/LibGfx/test-inputs/jbig2/json/bitmap-template1-tpgdon.json +++ b/Tests/LibGfx/test-inputs/jbig2/json/bitmap-template1-tpgdon.json @@ -12,7 +12,8 @@ "page_width": 399, "page_height": 400, "flags": { - "is_eventually_lossless": true + "is_eventually_lossless": true, + "default_combination_operator": "xor" } } }, @@ -21,6 +22,11 @@ "type": "lossless_generic_region", "page_association": 1, "data": { + "region_segment_information": { + "flags": { + "external_combination_operator": "xor" + } + }, "image_data": { "from_file": "bitmap.bmp" }, @@ -32,6 +38,47 @@ }, { "segment_number": 2, + "type": "lossless_generic_region", + "page_association": 1, + "data": { + "region_segment_information": { + "flags": { + "external_combination_operator": "xor" + } + }, + "image_data": { + "from_file": "tpgdon1.bmp", + "repeat_x": 30, + "repeat_y": 110 + }, + "flags": { + "gb_template": 1, + "use_typical_prediction": true + } + } + }, + { + "segment_number": 3, + "type": "lossless_generic_region", + "page_association": 1, + "data": { + "region_segment_information": { + "flags": { + "external_combination_operator": "xor" + } + }, + "image_data": { + "from_file": "tpgdon1.bmp", + "repeat_x": 30, + "repeat_y": 110 + }, + "flags": { + "use_typical_prediction": false + } + } + }, + { + "segment_number": 4, "type": "end_of_page", "page_association": 1 } diff --git a/Tests/LibGfx/test-inputs/jbig2/json/bitmap-template2-tpgdon.json b/Tests/LibGfx/test-inputs/jbig2/json/bitmap-template2-tpgdon.json index 89603210e3eae5..504bd15da370c9 100644 --- a/Tests/LibGfx/test-inputs/jbig2/json/bitmap-template2-tpgdon.json +++ b/Tests/LibGfx/test-inputs/jbig2/json/bitmap-template2-tpgdon.json @@ -12,7 +12,8 @@ "page_width": 399, "page_height": 400, "flags": { - "is_eventually_lossless": true + "is_eventually_lossless": true, + "default_combination_operator": "xor" } } }, @@ -21,6 +22,11 @@ "type": "lossless_generic_region", "page_association": 1, "data": { + "region_segment_information": { + "flags": { + "external_combination_operator": "xor" + } + }, "image_data": { "from_file": "bitmap.bmp" }, @@ -32,6 +38,47 @@ }, { "segment_number": 2, + "type": "lossless_generic_region", + "page_association": 1, + "data": { + "region_segment_information": { + "flags": { + "external_combination_operator": "xor" + } + }, + "image_data": { + "from_file": "tpgdon2.bmp", + "repeat_x": 30, + "repeat_y": 110 + }, + "flags": { + "gb_template": 2, + "use_typical_prediction": true + } + } + }, + { + "segment_number": 3, + "type": "lossless_generic_region", + "page_association": 1, + "data": { + "region_segment_information": { + "flags": { + "external_combination_operator": "xor" + } + }, + "image_data": { + "from_file": "tpgdon2.bmp", + "repeat_x": 30, + "repeat_y": 110 + }, + "flags": { + "use_typical_prediction": false + } + } + }, + { + "segment_number": 4, "type": "end_of_page", "page_association": 1 } diff --git a/Tests/LibGfx/test-inputs/jbig2/json/bitmap-template3-tpgdon.json b/Tests/LibGfx/test-inputs/jbig2/json/bitmap-template3-tpgdon.json index 0625d46b3a6aab..0912186501562d 100644 --- a/Tests/LibGfx/test-inputs/jbig2/json/bitmap-template3-tpgdon.json +++ b/Tests/LibGfx/test-inputs/jbig2/json/bitmap-template3-tpgdon.json @@ -12,7 +12,8 @@ "page_width": 399, "page_height": 400, "flags": { - "is_eventually_lossless": true + "is_eventually_lossless": true, + "default_combination_operator": "xor" } } }, @@ -21,6 +22,11 @@ "type": "lossless_generic_region", "page_association": 1, "data": { + "region_segment_information": { + "flags": { + "external_combination_operator": "xor" + } + }, "image_data": { "from_file": "bitmap.bmp" }, @@ -32,6 +38,47 @@ }, { "segment_number": 2, + "type": "lossless_generic_region", + "page_association": 1, + "data": { + "region_segment_information": { + "flags": { + "external_combination_operator": "xor" + } + }, + "image_data": { + "from_file": "tpgdon3.bmp", + "repeat_x": 30, + "repeat_y": 110 + }, + "flags": { + "gb_template": 3, + "use_typical_prediction": true + } + } + }, + { + "segment_number": 3, + "type": "lossless_generic_region", + "page_association": 1, + "data": { + "region_segment_information": { + "flags": { + "external_combination_operator": "xor" + } + }, + "image_data": { + "from_file": "tpgdon3.bmp", + "repeat_x": 30, + "repeat_y": 110 + }, + "flags": { + "use_typical_prediction": false + } + } + }, + { + "segment_number": 4, "type": "end_of_page", "page_association": 1 } diff --git a/Tests/LibGfx/test-inputs/jbig2/json/bitmap-tpgdon.json b/Tests/LibGfx/test-inputs/jbig2/json/bitmap-tpgdon.json index 335eb9b43e2118..b33073e3e92bf9 100644 --- a/Tests/LibGfx/test-inputs/jbig2/json/bitmap-tpgdon.json +++ b/Tests/LibGfx/test-inputs/jbig2/json/bitmap-tpgdon.json @@ -12,7 +12,8 @@ "page_width": 399, "page_height": 400, "flags": { - "is_eventually_lossless": true + "is_eventually_lossless": true, + "default_combination_operator": "xor" } } }, @@ -21,6 +22,11 @@ "type": "lossless_generic_region", "page_association": 1, "data": { + "region_segment_information": { + "flags": { + "external_combination_operator": "xor" + } + }, "image_data": { "from_file": "bitmap.bmp" }, @@ -31,6 +37,46 @@ }, { "segment_number": 2, + "type": "lossless_generic_region", + "page_association": 1, + "data": { + "region_segment_information": { + "flags": { + "external_combination_operator": "xor" + } + }, + "image_data": { + "from_file": "tpgdon0.bmp", + "repeat_x": 30, + "repeat_y": 110 + }, + "flags": { + "use_typical_prediction": true + } + } + }, + { + "segment_number": 3, + "type": "lossless_generic_region", + "page_association": 1, + "data": { + "region_segment_information": { + "flags": { + "external_combination_operator": "xor" + } + }, + "image_data": { + "from_file": "tpgdon0.bmp", + "repeat_x": 30, + "repeat_y": 110 + }, + "flags": { + "use_typical_prediction": false + } + } + }, + { + "segment_number": 4, "type": "end_of_page", "page_association": 1 } diff --git a/Tests/LibGfx/test-inputs/jbig2/json/compile.sh b/Tests/LibGfx/test-inputs/jbig2/json/compile.sh index 634db122257917..283daa1e4a43a5 100755 --- a/Tests/LibGfx/test-inputs/jbig2/json/compile.sh +++ b/Tests/LibGfx/test-inputs/jbig2/json/compile.sh @@ -1,16 +1,31 @@ #!/bin/bash -set -eu +set -u -DIR=$(dirname "$0") -LAGOM_BUILD=$DIR/../../../../../Build/lagom -JBIG2_FROM_JSON=$LAGOM_BUILD/bin/jbig2-from-json +DIR="$(cd "$(dirname "$0")" && pwd)" +ROOT="$DIR/../../../../.." +LAGOM_BUILD="$ROOT/Build/lagom" +JBIG2_FROM_JSON=${1:-"$LAGOM_BUILD/bin/jbig2-from-json"} -for f in "$DIR"/*.json; do - f_jb2="$DIR"/../$(basename "${f%.json}.jbig2") - if ! "$JBIG2_FROM_JSON" -o "$f_jb2" "$f"; then - echo failed to run: - echo "$JBIG2_FROM_JSON" -o "$f_jb2" "$f" - exit 1 - fi -done +export DIR +export JBIG2_FROM_JSON + +run_jbig2() { + f="$1" + filename=$(basename "${f%.json}.jbig2") + f_jb2="$DIR/../$filename" + + if ! output=$("$JBIG2_FROM_JSON" -o "$f_jb2" "$f" 2>&1); then + echo failed to run: + echo "$JBIG2_FROM_JSON" -o "$f_jb2" "$f" + echo "$output" + return 1 + fi +} + +. "$ROOT/Meta/shell_include.sh" +NPROC=$(get_number_of_processing_units) + +export -f run_jbig2 +find "$DIR" -maxdepth 1 -name "*.json" -print0 | \ + xargs -0 -P "$NPROC" -I {} bash -c 'run_jbig2 "$@"' _ {} diff --git a/Tests/LibGfx/test-inputs/jbig2/json/tpgdon0.bmp b/Tests/LibGfx/test-inputs/jbig2/json/tpgdon0.bmp new file mode 100644 index 00000000000000..462d67d230fa2f Binary files /dev/null and b/Tests/LibGfx/test-inputs/jbig2/json/tpgdon0.bmp differ diff --git a/Tests/LibGfx/test-inputs/jbig2/json/tpgdon1.bmp b/Tests/LibGfx/test-inputs/jbig2/json/tpgdon1.bmp new file mode 100644 index 00000000000000..4fe00cc9203bb4 Binary files /dev/null and b/Tests/LibGfx/test-inputs/jbig2/json/tpgdon1.bmp differ diff --git a/Tests/LibGfx/test-inputs/jbig2/json/tpgdon2.bmp b/Tests/LibGfx/test-inputs/jbig2/json/tpgdon2.bmp new file mode 100644 index 00000000000000..a3f15e40a7d859 Binary files /dev/null and b/Tests/LibGfx/test-inputs/jbig2/json/tpgdon2.bmp differ diff --git a/Tests/LibGfx/test-inputs/jbig2/json/tpgdon3.bmp b/Tests/LibGfx/test-inputs/jbig2/json/tpgdon3.bmp new file mode 100644 index 00000000000000..923efdec0f1a46 Binary files /dev/null and b/Tests/LibGfx/test-inputs/jbig2/json/tpgdon3.bmp differ diff --git a/Tests/LibGfx/test-inputs/jbig2/json/tpgron-ref.bmp b/Tests/LibGfx/test-inputs/jbig2/json/tpgron-ref.bmp new file mode 100644 index 00000000000000..633c0f9b30edba Binary files /dev/null and b/Tests/LibGfx/test-inputs/jbig2/json/tpgron-ref.bmp differ diff --git a/Tests/LibLine/CMakeLists.txt b/Tests/LibLine/CMakeLists.txt new file mode 100644 index 00000000000000..aca55ee6496e8f --- /dev/null +++ b/Tests/LibLine/CMakeLists.txt @@ -0,0 +1,7 @@ +set(TEST_SOURCES + TestMetrics.cpp +) + +foreach(source IN LISTS TEST_SOURCES) + serenity_test("${source}" LibLine LIBS LibLine LibUnicode) +endforeach() diff --git a/Tests/LibLine/TestMetrics.cpp b/Tests/LibLine/TestMetrics.cpp new file mode 100644 index 00000000000000..7d56243aea1de9 --- /dev/null +++ b/Tests/LibLine/TestMetrics.cpp @@ -0,0 +1,99 @@ +/* + * Copyright (c) 2025, the SerenityOS developers. + * + * SPDX-License-Identifier: BSD-2-Clause + */ + +#include + +#include + +TEST_CASE(count_ascii_glyphs_u8) +{ + constexpr auto string = "Hello, World!"sv; // length in bytes: 13, code points: 13, glyphs: 13 + auto metrics = Line::Editor::actual_rendered_string_metrics(string); + EXPECT_EQ(metrics.grapheme_breaks.size(), 13u); + EXPECT_EQ(metrics.line_metrics.size(), 1u); + EXPECT_EQ(metrics.line_metrics[0].length, 13u); + EXPECT_EQ(metrics.line_metrics[0].visible_length, 13u); +} + +TEST_CASE(count_ascii_glyphs_u32) +{ + constexpr u32 string[] = { 'H', 'e', 'l', 'l', 'o', ',', ' ', 'W', 'o', 'r', 'l', 'd', '!' }; // length in code points: 13, glyphs: 13 + auto metrics = Line::Editor::actual_rendered_string_metrics(Utf32View(string)); + EXPECT_EQ(metrics.grapheme_breaks.size(), 13u); + EXPECT_EQ(metrics.line_metrics.size(), 1u); + EXPECT_EQ(metrics.line_metrics[0].length, 13u); + EXPECT_EQ(metrics.line_metrics[0].visible_length, 13u); +} + +TEST_CASE(count_simple_multibyte_glyphs_u8) +{ + constexpr auto string = "Héllo, Wörld!"sv; // length in bytes: 15, code points: 13, glyphs: 13 + auto metrics = Line::Editor::actual_rendered_string_metrics(string); + EXPECT_EQ(metrics.grapheme_breaks.size(), 13u); + EXPECT_EQ(metrics.line_metrics.size(), 1u); + EXPECT_EQ(metrics.line_metrics[0].length, 13u); + EXPECT_EQ(metrics.line_metrics[0].visible_length, 13u); +} + +TEST_CASE(count_simple_multibyte_glyphs_u32) +{ + constexpr u32 string[] = { 'H', 0xe9, 'l', 'l', 'o', ',', ' ', 'W', 0xf6, 'r', 'l', 'd', '!' }; // length in code points: 13, glyphs: 13 + auto metrics = Line::Editor::actual_rendered_string_metrics(Utf32View(string)); + EXPECT_EQ(metrics.grapheme_breaks.size(), 13u); + EXPECT_EQ(metrics.line_metrics.size(), 1u); + EXPECT_EQ(metrics.line_metrics[0].length, 13u); + EXPECT_EQ(metrics.line_metrics[0].visible_length, 13u); +} + +TEST_CASE(count_multi_codepoint_glyphs_u8) +{ + constexpr auto string = "Héllo, Wörld! 👩‍💻"sv; // length in bytes: 25, code points: 17, glyphs: 15 + auto metrics = Line::Editor::actual_rendered_string_metrics(string); + EXPECT_EQ(metrics.grapheme_breaks.size(), 15u); + EXPECT_EQ(metrics.line_metrics.size(), 1u); + EXPECT_EQ(metrics.line_metrics[0].length, 17u); + EXPECT_EQ(metrics.line_metrics[0].visible_length, 17u); +} + +TEST_CASE(count_jp_glyphs_u8) +{ + { + constexpr auto string = "コンニチハ、ワールド！"sv; // length in bytes: 33, code points: 11, glyphs: 11 + auto metrics = Line::Editor::actual_rendered_string_metrics(string); + EXPECT_EQ(metrics.grapheme_breaks.size(), 11u); + EXPECT_EQ(metrics.line_metrics.size(), 1u); + EXPECT_EQ(metrics.line_metrics[0].length, 11u); + EXPECT_EQ(metrics.line_metrics[0].visible_length, 11u); + } + + { + constexpr auto string = "がぎぐげご"sv; // length in bytes: 18, code points: 10, glyphs: 5 + auto metrics = Line::Editor::actual_rendered_string_metrics(string); + EXPECT_EQ(metrics.grapheme_breaks.size(), 5u); + EXPECT_EQ(metrics.line_metrics.size(), 1u); + EXPECT_EQ(metrics.line_metrics[0].length, 10u); + EXPECT_EQ(metrics.line_metrics[0].visible_length, 10u); + } + + { + constexpr auto string = "食べる"sv; // length in bytes: 12, code points: 4, glyphs: 3 + auto metrics = Line::Editor::actual_rendered_string_metrics(string); + EXPECT_EQ(metrics.grapheme_breaks.size(), 3u); + EXPECT_EQ(metrics.line_metrics.size(), 1u); + EXPECT_EQ(metrics.line_metrics[0].length, 4u); + EXPECT_EQ(metrics.line_metrics[0].visible_length, 4u); + } +} + +TEST_CASE(count_multi_codepoint_glyphs_mixed_u8) +{ + constexpr auto string = "Héllo, コンニチハ! 👩‍💻 persian word: کتاب"sv; // length in bytes: 59, code points: 36, glyphs: 34 + auto metrics = Line::Editor::actual_rendered_string_metrics(string); + EXPECT_EQ(metrics.grapheme_breaks.size(), 34u); + EXPECT_EQ(metrics.line_metrics.size(), 1u); + EXPECT_EQ(metrics.line_metrics[0].length, 36u); + EXPECT_EQ(metrics.line_metrics[0].visible_length, 36u); +} diff --git a/Tests/LibPDF/complex.pdf b/Tests/LibPDF/complex.pdf index 4d7a58dcb6fc6b..c13dea37be28e6 100644 Binary files a/Tests/LibPDF/complex.pdf and b/Tests/LibPDF/complex.pdf differ diff --git a/Tests/LibRegex/Regex.cpp b/Tests/LibRegex/Regex.cpp index 901238a24b5a7d..338648aeba65f5 100644 --- a/Tests/LibRegex/Regex.cpp +++ b/Tests/LibRegex/Regex.cpp @@ -949,6 +949,8 @@ TEST_CASE(replace) { "foo(.+)"sv, "\\2\\1"sv, "foobar"sv, "\\2bar"sv }, { "foo(.+)"sv, "\\\\\\1"sv, "foobar"sv, "\\bar"sv }, { "foo(.)"sv, "a\\1"sv, "fooxfooy"sv, "axay"sv, ECMAScriptFlags::Multiline }, + { "(.)(.*)"sv, "\\&\\0\\1\\2&\\&"sv, "abc"sv, "&abcabcabc&"sv }, + { ".*"sv, "&\\0"sv, "foo"sv, "foofoo"sv }, }; for (auto& test : tests) { diff --git a/Tests/Utilities/TestSed.cpp b/Tests/Utilities/TestSed.cpp index 825764b5bf9601..d49f9190cf3e9d 100644 --- a/Tests/Utilities/TestSed.cpp +++ b/Tests/Utilities/TestSed.cpp @@ -77,3 +77,10 @@ TEST_CASE(complex) { run_sed({ "h; x; s/./*/gp; x; h; p; x; s/./*/gp", "-n" }, "hello serenity"sv, "**************\nhello serenity\n**************\n"sv); } + +TEST_CASE(strict_posix) +{ + run_sed({ "--posix", "s/(.)/b/" }, "foo\n"sv, "foo\n"sv); + run_sed({ "--posix", "s/\\(.\\)/b/" }, "foo\n"sv, "boo\n"sv); + run_sed({ "--posix", "s/\\(.\\)\\(.*\\)/\\1\\2\\1/" }, "foo\n"sv, "foof\n"sv); +} diff --git a/Toolchain/BuildJakt.sh b/Toolchain/BuildJakt.sh index c0d0d00a8de635..4703a4f4ab5a14 100755 --- a/Toolchain/BuildJakt.sh +++ b/Toolchain/BuildJakt.sh @@ -18,39 +18,30 @@ PREFIX="$DIR/Local/jakt" VALID_TOOLCHAINS=() -declare -A CXX_GNU=() -declare -A CXX_CLANG=() -declare -A RANLIB_GNU=() -declare -A RANLIB_CLANG=() -declare -A BUILD_GNU=() -declare -A BUILD_CLANG=() -: "${BUILD_GNU[@]}" "${BUILD_CLANG[@]}" # make shellcheck understand that these might be used. - for ARCH in "${ARCHES[@]}"; do TARGET="$ARCH-pc-serenity" - BUILD_GNU["$ARCH"]="$DIR/../Build/$ARCH" - BUILD_CLANG["$ARCH"]="$DIR/../Build/${ARCH}clang" + eval "BUILD_GNU_${ARCH}=\"$DIR/../Build/$ARCH\"" + eval "BUILD_CLANG_${ARCH}=\"$DIR/../Build/${ARCH}clang\"" - CXX_GNU["$ARCH"]="$DIR/Local/$ARCH/bin/$TARGET-g++" - CXX_CLANG["$ARCH"]="$DIR/Local/clang/bin/$TARGET-clang++" + eval "CXX_GNU_${ARCH}=\"$DIR/Local/$ARCH/bin/$TARGET-g++\"" + eval "CXX_CLANG_${ARCH}=\"$DIR/Local/clang/bin/$TARGET-clang++\"" - # Indirectly accessed below. - # shellcheck disable=SC2034 - RANLIB_GNU["$ARCH"]="$DIR/Local/$ARCH/bin/$TARGET-ranlib" - # shellcheck disable=SC2034 - RANLIB_CLANG["$ARCH"]="$DIR/Local/clang/bin/llvm-ranlib" + eval "RANLIB_GNU_${ARCH}=\"$DIR/Local/$ARCH/bin/$TARGET-ranlib\"" + eval "RANLIB_CLANG_${ARCH}=\"$DIR/Local/clang/bin/llvm-ranlib\"" - if [ -x "${CXX_GNU["$ARCH"]}" ]; then + VAR_CXX_GNU="CXX_GNU_${ARCH}" + if [ -x "${!VAR_CXX_GNU}" ]; then VALID_TOOLCHAINS+=("GNU;${ARCH}") fi - if [ -x "${CXX_CLANG[${ARCH}]}" ]; then + VAR_CXX_CLANG="CXX_CLANG_${ARCH}" + if [ -x "${!VAR_CXX_CLANG}" ]; then VALID_TOOLCHAINS+=("CLANG;${ARCH}") fi done -if [ "$FINAL_TARGET" = serenity ] && [ "${#VALID_TOOLCHAINS}" -eq 0 ]; then +if [ "$FINAL_TARGET" = serenity ] && [ "${#VALID_TOOLCHAINS[@]}" -eq 0 ]; then die "Need to build at least one C++ toolchain (either GNU or Clang) before BuildJakt.sh can succeed" fi @@ -62,6 +53,8 @@ NPROC=$(get_number_of_processing_units) if [ "$SYSTEM_NAME" = "OpenBSD" ]; then REALPATH="readlink -f" export CXX=eg++ +elif [ "$SYSTEM_NAME" = "Darwin" ]; then + REALPATH="perl -e 'use Cwd \"abs_path\"; print abs_path(shift)'" fi if command -v ginstall &>/dev/null; then @@ -89,7 +82,7 @@ buildstep_ninja() { mkdir -p "$DIR/Tarballs" -JAKT_COMMIT_HASH="bf6e9ce89206fb744d8acc6e700e31e4330a5f25" +JAKT_COMMIT_HASH="e990cc5667e32bc295556057972f117662b009bb" JAKT_NAME="jakt-${JAKT_COMMIT_HASH}" JAKT_TARBALL="${JAKT_NAME}.tar.gz" JAKT_GIT_URL="https://github.com/serenityos/jakt" @@ -198,9 +191,9 @@ build_for() { TARGET="$ARCH-pc-serenity" JAKT_TARGET="$TARGET-unknown" - current_build="BUILD_${TOOLCHAIN}[${ARCH}]" - current_cxx="CXX_${TOOLCHAIN}[${ARCH}]" - current_ranlib="RANLIB_${TOOLCHAIN}[${ARCH}]" + current_build="BUILD_${TOOLCHAIN}_${ARCH}" + current_cxx="CXX_${TOOLCHAIN}_${ARCH}" + current_ranlib="RANLIB_${TOOLCHAIN}_${ARCH}" BUILD="${!current_build}" TARGET_CXX="${!current_cxx}" @@ -214,7 +207,7 @@ build_for() { if [ ! -d "$BUILD" ]; then mkdir -p "$BUILD" fi - BUILD=$($REALPATH "$BUILD") + BUILD=$(eval "$REALPATH" "$BUILD") echo "XXX building jakt support libs in $BUILD with $TARGET_CXX" SYSROOT="$BUILD/Root" @@ -227,7 +220,7 @@ build_for() { mkdir -p "$BUILD" pushd "$BUILD" mkdir -p Root/usr/include/ - SRC_ROOT=$($REALPATH "$DIR"/..) + SRC_ROOT=$(eval "$REALPATH" "$DIR"/..) FILES=$(find \ "$SRC_ROOT"/AK \ "$SRC_ROOT"/Kernel/API \ diff --git a/Userland/Applications/Maps/SearchPanel.cpp b/Userland/Applications/Maps/SearchPanel.cpp index bd4251562aecc9..65e84f096a0eeb 100644 --- a/Userland/Applications/Maps/SearchPanel.cpp +++ b/Userland/Applications/Maps/SearchPanel.cpp @@ -84,9 +84,23 @@ void SearchPanel::search(StringView query) // FIXME: Handle JSON parsing errors auto const& json_place = json_places.at(i).as_object(); - MapWidget::LatLng latlng = { json_place.get_byte_string("lat"sv).release_value().to_number().release_value(), - json_place.get_byte_string("lon"sv).release_value().to_number().release_value() }; - String name = MUST(String::formatted("{}\n{:.5}, {:.5}", json_place.get_byte_string("display_name"sv).release_value(), latlng.latitude, latlng.longitude)); + auto lat_string = json_place.get_byte_string("lat"sv); + auto lon_string = json_place.get_byte_string("lon"sv); + auto display_name = json_place.get_byte_string("display_name"sv); + + if (!lat_string.has_value() || !lon_string.has_value() || !display_name.has_value()) { + continue; + } + + auto lat_number = lat_string.release_value().to_number(); + auto lon_number = lon_string.release_value().to_number(); + + if (!lat_number.has_value() || !lon_number.has_value()) { + continue; + } + + MapWidget::LatLng latlng = { lat_number.release_value(), lon_number.release_value() }; + String name = MUST(String::formatted("{}\n{:.5}, {:.5}", display_name.release_value(), latlng.latitude, latlng.longitude)); // Calculate the right zoom level for bounding box auto const& json_boundingbox = json_place.get_array("boundingbox"sv); diff --git a/Userland/Applications/PixelPaint/Filters/Median.cpp b/Userland/Applications/PixelPaint/Filters/Median.cpp index af4ace4e70afe1..26781ae6f4b4bc 100644 --- a/Userland/Applications/PixelPaint/Filters/Median.cpp +++ b/Userland/Applications/PixelPaint/Filters/Median.cpp @@ -31,7 +31,7 @@ void Median::apply(Gfx::Bitmap& target_bitmap, Gfx::Bitmap const& source_bitmap) values.unchecked_append(source_bitmap.get_pixel(i, j)); } } - // FIXME: If there was an insertion sort in AK, we should better use that here. + // Sort the values to be able to extract the median. The median is determined by grey value (luminosity). quick_sort(values, [](auto& a, auto& b) { return a.luminosity() < b.luminosity(); }); target->set_pixel(x, y, values[values.size() / 2]); diff --git a/Userland/Applications/PixelPaint/MainWidget.cpp b/Userland/Applications/PixelPaint/MainWidget.cpp index 89da0de2ec3b93..fc949330567165 100644 --- a/Userland/Applications/PixelPaint/MainWidget.cpp +++ b/Userland/Applications/PixelPaint/MainWidget.cpp @@ -735,6 +735,18 @@ ErrorOr MainWidget::initialize_menubar(GUI::Window& window) editor->did_complete_action("Crop Image to Content"sv); })); + m_image_menu->add_separator(); + + m_levels_dialog_action = GUI::Action::create( + "Change &Levels...", { Mod_Ctrl, Key_L }, g_icon_bag.levels, [&](auto&) { + auto* editor = current_image_editor(); + VERIFY(editor); + auto dialog = PixelPaint::LevelsDialog::construct(&window, editor); + if (dialog->exec() != GUI::Dialog::ExecResult::OK) + dialog->revert_possible_changes(); + }); + m_image_menu->add_action(*m_levels_dialog_action); + m_layer_menu = window.add_menu("&Layer"_string); m_layer_menu->on_visibility_change = [this](bool visible) { @@ -1193,15 +1205,6 @@ ErrorOr MainWidget::initialize_menubar(GUI::Window& window) })); help_menu->add_action(GUI::CommonActions::make_about_action("Pixel Paint"_string, GUI::Icon::default_icon("app-pixel-paint"sv), &window)); - m_levels_dialog_action = GUI::Action::create( - "Change &Levels...", { Mod_Ctrl, Key_L }, g_icon_bag.levels, [&](auto&) { - auto* editor = current_image_editor(); - VERIFY(editor); - auto dialog = PixelPaint::LevelsDialog::construct(&window, editor); - if (dialog->exec() != GUI::Dialog::ExecResult::OK) - dialog->revert_possible_changes(); - }); - auto& toolbar = *find_descendant_of_type_named("toolbar"); toolbar.add_action(*m_new_image_action); toolbar.add_action(*m_open_image_action); diff --git a/Userland/Applications/SpaceAnalyzer/TreeMapWidget.cpp b/Userland/Applications/SpaceAnalyzer/TreeMapWidget.cpp index f39390849128b3..06bba0da2a0f7b 100644 --- a/Userland/Applications/SpaceAnalyzer/TreeMapWidget.cpp +++ b/Userland/Applications/SpaceAnalyzer/TreeMapWidget.cpp @@ -15,7 +15,6 @@ #include #include #include -#include namespace SpaceAnalyzer { diff --git a/Userland/Libraries/LibC/fcntl.cpp b/Userland/Libraries/LibC/fcntl.cpp index 4d3a9b40960fb1..4de774cf891e4e 100644 --- a/Userland/Libraries/LibC/fcntl.cpp +++ b/Userland/Libraries/LibC/fcntl.cpp @@ -6,7 +6,6 @@ */ #include -#include #include #include #include @@ -120,65 +119,4 @@ int posix_fallocate(int fd, off_t offset, off_t len) // posix_fallocate does not set errno. return -static_cast(syscall(SC_posix_fallocate, fd, offset, len)); } - -// https://pubs.opengroup.org/onlinepubs/9699919799/functions/utimensat.html -int utimensat(int dirfd, char const* path, struct timespec const times[2], int flag) -{ - if (!path) { - errno = EFAULT; - return -1; - } - return __utimens(dirfd, path, times, flag); -} - -int __utimens(int fd, char const* path, struct timespec const times[2], int flag) -{ - size_t path_length = 0; - if (path) { - path_length = strlen(path); - if (path_length > INT32_MAX) { - errno = EINVAL; - return -1; - } - } - - // POSIX allows AT_SYMLINK_NOFOLLOW flag or no flags. - if (flag & ~AT_SYMLINK_NOFOLLOW) { - errno = EINVAL; - return -1; - } - - // Return early without error since both changes are to be omitted. - if (times && times[0].tv_nsec == UTIME_OMIT && times[1].tv_nsec == UTIME_OMIT) - return 0; - - // According to POSIX, when times is a nullptr, it's equivalent to setting - // both last access time and last modification time to the current time. - // Setting the times argument to nullptr if it matches this case prevents - // the need to copy it in the kernel. - if (times && times[0].tv_nsec == UTIME_NOW && times[1].tv_nsec == UTIME_NOW) - times = nullptr; - - if (times) { - for (int i = 0; i < 2; ++i) { - if ((times[i].tv_nsec != UTIME_NOW && times[i].tv_nsec != UTIME_OMIT) - && (times[i].tv_nsec < 0 || times[i].tv_nsec >= 1'000'000'000L)) { - errno = EINVAL; - return -1; - } - } - } - - int rc = 0; - if (path) { - // NOTE: fd is treated as dirfd for this syscall. - Syscall::SC_utimensat_params params { fd, { path, path_length }, times, flag }; - rc = syscall(SC_utimensat, ¶ms); - } else { - Syscall::SC_futimens_params params { fd, times }; - rc = syscall(SC_futimens, ¶ms); - } - - __RETURN_WITH_ERRNO(rc, rc, -1); -} } diff --git a/Userland/Libraries/LibC/fcntl.h b/Userland/Libraries/LibC/fcntl.h index ceb5896c474d2e..e1127b96d7e409 100644 --- a/Userland/Libraries/LibC/fcntl.h +++ b/Userland/Libraries/LibC/fcntl.h @@ -37,6 +37,4 @@ int inode_watcher_remove_watch(int fd, int wd); int posix_fadvise(int fd, off_t offset, off_t len, int advice); int posix_fallocate(int fd, off_t offset, off_t len); -int utimensat(int dirfd, char const* path, struct timespec const times[2], int flag); - __END_DECLS diff --git a/Userland/Libraries/LibC/math.h b/Userland/Libraries/LibC/math.h index baee831209a3eb..8ed309c3100b3d 100644 --- a/Userland/Libraries/LibC/math.h +++ b/Userland/Libraries/LibC/math.h @@ -26,7 +26,7 @@ __BEGIN_DECLS #define HUGE_VAL __builtin_huge_val() #define HUGE_VALL __builtin_huge_vall() #define INFINITY __builtin_huge_valf() -#define NAN __builtin_nan("") +#define NAN __builtin_nanf("") #define MAXFLOAT FLT_MAX #define M_El 2.718281828459045235360287471352662498L diff --git a/Userland/Libraries/LibC/stat.cpp b/Userland/Libraries/LibC/stat.cpp index 769634b971a0e2..5b5236131c2f45 100644 --- a/Userland/Libraries/LibC/stat.cpp +++ b/Userland/Libraries/LibC/stat.cpp @@ -87,6 +87,24 @@ int mkfifoat(int dirfd, char const* pathname, mode_t mode) return mknodat(dirfd, pathname, mode | S_IFIFO, 0); } +// https://pubs.opengroup.org/onlinepubs/9699919799/functions/mknod.html +int mknod(char const* pathname, mode_t mode, dev_t dev) +{ + return mknodat(AT_FDCWD, pathname, mode, dev); +} + +// https://pubs.opengroup.org/onlinepubs/9699919799/functions/mknodat.html +int mknodat(int dirfd, char const* pathname, mode_t mode, dev_t dev) +{ + if (!pathname) { + errno = EFAULT; + return -1; + } + Syscall::SC_mknod_params params { { pathname, strlen(pathname) }, mode, dev, dirfd }; + int rc = syscall(SC_mknod, ¶ms); + __RETURN_WITH_ERRNO(rc, rc, -1); +} + static int do_stat(int dirfd, char const* path, struct stat* statbuf, bool follow_symlinks) { if (!path) { @@ -128,4 +146,65 @@ int futimens(int fd, struct timespec const times[2]) { return __utimens(fd, nullptr, times, 0); } + +// https://pubs.opengroup.org/onlinepubs/9699919799/functions/utimensat.html +int utimensat(int dirfd, char const* path, struct timespec const times[2], int flag) +{ + if (!path) { + errno = EFAULT; + return -1; + } + return __utimens(dirfd, path, times, flag); +} + +int __utimens(int fd, char const* path, struct timespec const times[2], int flag) +{ + size_t path_length = 0; + if (path) { + path_length = strlen(path); + if (path_length > INT32_MAX) { + errno = EINVAL; + return -1; + } + } + + // POSIX allows AT_SYMLINK_NOFOLLOW flag or no flags. + if (flag & ~AT_SYMLINK_NOFOLLOW) { + errno = EINVAL; + return -1; + } + + // Return early without error since both changes are to be omitted. + if (times && times[0].tv_nsec == UTIME_OMIT && times[1].tv_nsec == UTIME_OMIT) + return 0; + + // According to POSIX, when times is a nullptr, it's equivalent to setting + // both last access time and last modification time to the current time. + // Setting the times argument to nullptr if it matches this case prevents + // the need to copy it in the kernel. + if (times && times[0].tv_nsec == UTIME_NOW && times[1].tv_nsec == UTIME_NOW) + times = nullptr; + + if (times) { + for (int i = 0; i < 2; ++i) { + if ((times[i].tv_nsec != UTIME_NOW && times[i].tv_nsec != UTIME_OMIT) + && (times[i].tv_nsec < 0 || times[i].tv_nsec >= 1'000'000'000L)) { + errno = EINVAL; + return -1; + } + } + } + + int rc = 0; + if (path) { + // NOTE: fd is treated as dirfd for this syscall. + Syscall::SC_utimensat_params params { fd, { path, path_length }, times, flag }; + rc = syscall(SC_utimensat, ¶ms); + } else { + Syscall::SC_futimens_params params { fd, times }; + rc = syscall(SC_futimens, ¶ms); + } + + __RETURN_WITH_ERRNO(rc, rc, -1); +} } diff --git a/Userland/Libraries/LibC/sys/stat.h b/Userland/Libraries/LibC/sys/stat.h index 67417a11ef9731..bee3e6c2d8dbde 100644 --- a/Userland/Libraries/LibC/sys/stat.h +++ b/Userland/Libraries/LibC/sys/stat.h @@ -21,10 +21,13 @@ int mkdir(char const* pathname, mode_t); int mkdirat(int dirfd, char const* pathname, mode_t); int mkfifo(char const* pathname, mode_t); int mkfifoat(int dirfd, char const* pathname, mode_t); +int mknod(char const* pathname, mode_t, dev_t); +int mknodat(int dirfd, char const* pathname, mode_t, dev_t); int fstat(int fd, struct stat* statbuf); int lstat(char const* path, struct stat* statbuf); int stat(char const* path, struct stat* statbuf); int fstatat(int fd, char const* path, struct stat* statbuf, int flags); int futimens(int fd, struct timespec const times[2]); +int utimensat(int dirfd, char const* path, struct timespec const times[2], int flag); __END_DECLS diff --git a/Userland/Libraries/LibC/unistd.cpp b/Userland/Libraries/LibC/unistd.cpp index a8125fe3c2c392..1f9ab668fc41db 100644 --- a/Userland/Libraries/LibC/unistd.cpp +++ b/Userland/Libraries/LibC/unistd.cpp @@ -154,16 +154,16 @@ int execv(char const* path, char* const argv[]) int execve(char const* filename, char* const argv[], char* const envp[]) { size_t arg_count = 0; - for (size_t i = 0; argv[i]; ++i) + for (size_t i = 0; argv && argv[i]; ++i) ++arg_count; size_t env_count = 0; - for (size_t i = 0; envp[i]; ++i) + for (size_t i = 0; envp && envp[i]; ++i) ++env_count; auto copy_strings = [&](auto& vec, size_t count, auto& output) { output.length = count; - for (size_t i = 0; vec[i]; ++i) { + for (size_t i = 0; i < count; ++i) { output.strings[i].characters = vec[i]; output.strings[i].length = strlen(vec[i]); } @@ -826,24 +826,6 @@ int faccessat(int dirfd, char const* pathname, int mode, int flags) __RETURN_WITH_ERRNO(rc, rc, -1); } -// https://pubs.opengroup.org/onlinepubs/9699919799/functions/mknod.html -int mknod(char const* pathname, mode_t mode, dev_t dev) -{ - return mknodat(AT_FDCWD, pathname, mode, dev); -} - -// https://pubs.opengroup.org/onlinepubs/9699919799/functions/mknodat.html -int mknodat(int dirfd, char const* pathname, mode_t mode, dev_t dev) -{ - if (!pathname) { - errno = EFAULT; - return -1; - } - Syscall::SC_mknod_params params { { pathname, strlen(pathname) }, mode, dev, dirfd }; - int rc = syscall(SC_mknod, ¶ms); - __RETURN_WITH_ERRNO(rc, rc, -1); -} - // https://pubs.opengroup.org/onlinepubs/9699919799/functions/fpathconf.html long fpathconf([[maybe_unused]] int fd, int name) { diff --git a/Userland/Libraries/LibC/unistd.h b/Userland/Libraries/LibC/unistd.h index 74d4369ba5076d..7a6bb2c2af57e8 100644 --- a/Userland/Libraries/LibC/unistd.h +++ b/Userland/Libraries/LibC/unistd.h @@ -107,8 +107,6 @@ unsigned int alarm(unsigned int seconds); int access(char const* pathname, int mode); int faccessat(int dirfd, char const* pathname, int mode, int flags); int isatty(int fd); -int mknod(char const* pathname, mode_t, dev_t); -int mknodat(int dirfd, char const* pathname, mode_t, dev_t); long fpathconf(int fd, int name); long pathconf(char const* path, int name); char* getlogin(void); diff --git a/Userland/Libraries/LibCore/System.cpp b/Userland/Libraries/LibCore/System.cpp index 506e81a35428d3..6f793382677b48 100644 --- a/Userland/Libraries/LibCore/System.cpp +++ b/Userland/Libraries/LibCore/System.cpp @@ -672,7 +672,10 @@ ErrorOr lstat(StringView path) ErrorOr read(int fd, Bytes buffer) { - ssize_t rc = ::read(fd, buffer.data(), buffer.size()); + ssize_t rc; + do { + rc = ::read(fd, buffer.data(), buffer.size()); + } while (rc < 0 && errno == EINTR); if (rc < 0) return Error::from_syscall("read"sv, -errno); return rc; @@ -680,7 +683,10 @@ ErrorOr read(int fd, Bytes buffer) ErrorOr write(int fd, ReadonlyBytes buffer) { - ssize_t rc = ::write(fd, buffer.data(), buffer.size()); + ssize_t rc; + do { + rc = ::write(fd, buffer.data(), buffer.size()); + } while (rc < 0 && errno == EINTR); if (rc < 0) return Error::from_syscall("write"sv, -errno); return rc; diff --git a/Userland/Libraries/LibCrypto/CMakeLists.txt b/Userland/Libraries/LibCrypto/CMakeLists.txt index 71c63211a96bb1..947a9aca34a6d0 100644 --- a/Userland/Libraries/LibCrypto/CMakeLists.txt +++ b/Userland/Libraries/LibCrypto/CMakeLists.txt @@ -20,7 +20,6 @@ set(SOURCES Checksum/Adler32.cpp Checksum/cksum.cpp Checksum/CRC32.cpp - Checksum/IPv4Header.cpp Cipher/AES.cpp Cipher/ChaCha20.cpp Curves/Curve25519.cpp diff --git a/Userland/Libraries/LibCrypto/Checksum/IPv4Header.cpp b/Userland/Libraries/LibCrypto/Checksum/IPv4Header.cpp deleted file mode 100644 index d731a2ccc4a98c..00000000000000 --- a/Userland/Libraries/LibCrypto/Checksum/IPv4Header.cpp +++ /dev/null @@ -1,40 +0,0 @@ -/* - * Copyright (c) 2020-2022, the SerenityOS developers. - * - * SPDX-License-Identifier: BSD-2-Clause - */ - -#include -#include -#include -#include -#include -#include -#include -#include - -namespace Crypto::Checksum { - -void IPv4Header::update(ReadonlyBytes data) -{ - for (size_t i = 0; i < data.size() / sizeof(u16); ++i) - // Dealing with the byte order isn't technically a part of the checksumming - // process, because it's expected that you'd already have the packet decoded, - // but since we're dealing with raw data, the responsibility of decoding (also) - // falls on us. - m_state += AK::convert_between_host_and_network_endian(ByteReader::load16(data.offset_pointer(i * sizeof(u16)))); -} - -u16 IPv4Header::digest() -{ - u32 output = m_state; - // While there are any bits above the bottom 16... - while (output >> 16) - // Drop the top bits, and add the carries to the sum. - output = (output & 0xFFFF) + (output >> 16); - - // Return the one's complement. - return htons(~static_cast(output)); -} - -} diff --git a/Userland/Libraries/LibCrypto/Checksum/IPv4Header.h b/Userland/Libraries/LibCrypto/Checksum/IPv4Header.h deleted file mode 100644 index 1a53f59208fc96..00000000000000 --- a/Userland/Libraries/LibCrypto/Checksum/IPv4Header.h +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Copyright (c) 2025, the SerenityOS developers. - * - * SPDX-License-Identifier: BSD-2-Clause - */ - -#pragma once - -#include -#include -#include - -namespace Crypto::Checksum { - -class IPv4Header : public ChecksumFunction { -public: - IPv4Header() = default; - IPv4Header(ReadonlyBytes data) - { - update(data); - } - - virtual void update(ReadonlyBytes data) override; - virtual u16 digest() override; - -private: - // NOTE: This is intentionally 32-bit rather than 16-bit. - // When we're in the process of summing up, we need to avoid - // overflowing or otherwise losing the carries in the process. - u32 m_state { 0 }; -}; - -} diff --git a/Userland/Libraries/LibDeviceTree/DeviceTree.cpp b/Userland/Libraries/LibDeviceTree/DeviceTree.cpp index 86188eb4d8bbb3..d8682d31812d21 100644 --- a/Userland/Libraries/LibDeviceTree/DeviceTree.cpp +++ b/Userland/Libraries/LibDeviceTree/DeviceTree.cpp @@ -161,11 +161,11 @@ ErrorOr> Node::interrupts(DeviceTree const& device_tree) const auto interrupt_cells = interrupt_cells_prop->as(); - auto interrupt_identifier = TRY(stream.read_in_place(interrupt_cells * sizeof(u32))); + auto interrupt_specifier = TRY(stream.read_in_place(interrupt_cells * sizeof(u32))); TRY(interrupts.try_append(Interrupt { .domain_root = &domain_root, - .interrupt_identifier = interrupt_identifier, + .interrupt_specifier = interrupt_specifier, })); } @@ -200,7 +200,7 @@ ErrorOr> Node::interrupts(DeviceTree const& device_tree) const for (size_t i = 0; i < interrupt_count; i++) { interrupts[i] = Interrupt { .domain_root = &domain_root, - .interrupt_identifier = interrupts_raw.slice(i * interrupt_cells * sizeof(u32), interrupt_cells * sizeof(u32)), + .interrupt_specifier = interrupts_raw.slice(i * interrupt_cells * sizeof(u32), interrupt_cells * sizeof(u32)), }; } @@ -339,7 +339,7 @@ ErrorOr
Ranges::translate_child_bus_address_to_parent_bus_address(Addre // If the property is defined with an value, it specifies that the parent and child address space is identical, // and no address translation is required. if (entry_count() == 0) - return addr; + return addr.clone().release_value_but_fixme_should_propagate_errors(); for (size_t i = 0; i < entry_count(); i++) { if (auto translation_or_error = MUST(entry(i)).translate_child_bus_address_to_parent_bus_address(addr); !translation_or_error.is_error()) @@ -354,7 +354,7 @@ ErrorOr
Node::translate_child_bus_address_to_root_address(Address const dbgln_if(DEVICETREE_DEBUG, "DeviceTree: Translating bus address {:hex-dump}", addr.raw()); auto const* current_node = this; - auto current_address = addr; + auto current_address = addr.clone().release_value_but_fixme_should_propagate_errors(); while (!current_node->is_root()) { // 2.3.8 ranges diff --git a/Userland/Libraries/LibDeviceTree/DeviceTree.h b/Userland/Libraries/LibDeviceTree/DeviceTree.h index acef374e0ba046..18a29f4945bd1b 100644 --- a/Userland/Libraries/LibDeviceTree/DeviceTree.h +++ b/Userland/Libraries/LibDeviceTree/DeviceTree.h @@ -28,10 +28,15 @@ class Address { public: Address() = default; Address(ReadonlyBytes data) - : m_raw(static_cast(data)) + : m_raw(decltype(m_raw)::from_span(data).release_value_but_fixme_should_propagate_errors()) { } + ErrorOr
clone() const + { + return Address { TRY(m_raw.clone()) }; + } + ReadonlyBytes raw() const { return m_raw; } template @@ -51,7 +56,7 @@ class Address { BigEndian big_endian_flatptr { flatptr }; Address address; - address.m_raw.resize(sizeof(FlatPtr)); + address.m_raw.try_resize(sizeof(FlatPtr)).release_value_but_fixme_should_propagate_errors(); __builtin_memcpy(address.m_raw.data(), &big_endian_flatptr, sizeof(big_endian_flatptr)); return address; } @@ -73,8 +78,13 @@ class Size { public: Size() = default; Size(ReadonlyBytes data) - : m_raw(static_cast(data)) + : m_raw(decltype(m_raw)::from_span(data).release_value_but_fixme_should_propagate_errors()) + { + } + + ErrorOr clone() const { + return Size { TRY(m_raw.clone()) }; } ReadonlyBytes raw() const { return m_raw; } @@ -94,7 +104,7 @@ class Size { struct Interrupt { Node const* domain_root; - ReadonlyBytes interrupt_identifier; + ReadonlyBytes interrupt_specifier; }; struct Property { @@ -178,8 +188,8 @@ struct Property { class RegEntry { public: RegEntry(Address const& address, Size const& size, Node const& node) - : m_address(address) - , m_length(size) + : m_address(address.clone().release_value_but_fixme_should_propagate_errors()) + , m_length(size.clone().release_value_but_fixme_should_propagate_errors()) , m_node(node) { } @@ -191,8 +201,8 @@ class RegEntry { { } - Address bus_address() const { return m_address; } - Size length() const { return m_length; } + Address bus_address() const { return m_address.clone().release_value_but_fixme_should_propagate_errors(); } + Size length() const { return m_length.clone().release_value_but_fixme_should_propagate_errors(); } ErrorOr
resolve_root_address() const; @@ -222,9 +232,9 @@ class Reg { class RangesEntry { public: RangesEntry(Address const& child_bus_address, Address const& parent_bus_address, Size const& length, Node const& node) - : m_child_bus_address(child_bus_address) - , m_parent_bus_address(parent_bus_address) - , m_length(length) + : m_child_bus_address(child_bus_address.clone().release_value_but_fixme_should_propagate_errors()) + , m_parent_bus_address(parent_bus_address.clone().release_value_but_fixme_should_propagate_errors()) + , m_length(length.clone().release_value_but_fixme_should_propagate_errors()) , m_node(node) { } @@ -237,9 +247,9 @@ class RangesEntry { { } - Address child_bus_address() const { return m_child_bus_address; } - Address parent_bus_address() const { return m_parent_bus_address; } - Size length() const { return m_length; } + Address child_bus_address() const { return m_child_bus_address.clone().release_value_but_fixme_should_propagate_errors(); } + Address parent_bus_address() const { return m_parent_bus_address.clone().release_value_but_fixme_should_propagate_errors(); } + Size length() const { return m_length.clone().release_value_but_fixme_should_propagate_errors(); } ErrorOr
translate_child_bus_address_to_parent_bus_address(Address const&) const; diff --git a/Userland/Libraries/LibEDID/EDID.cpp b/Userland/Libraries/LibEDID/EDID.cpp index c87e686cb96d6e..ddd1e4e398404b 100644 --- a/Userland/Libraries/LibEDID/EDID.cpp +++ b/Userland/Libraries/LibEDID/EDID.cpp @@ -1054,7 +1054,11 @@ auto Parser::supported_resolutions() const -> ErrorOr= 1); + resolution.refresh_rates.unchecked_append({ refresh_rate, preferred }); + + resolutions.try_append(move(resolution)).release_value_but_fixme_should_propagate_errors(); } else { auto& info = *it; SupportedResolution::RefreshRate* found_refresh_rate = nullptr; @@ -1067,7 +1071,7 @@ auto Parser::supported_resolutions() const -> ErrorOrpreferred |= preferred; else - info.refresh_rates.append({ refresh_rate, preferred }); + info.refresh_rates.try_append({ refresh_rate, preferred }).release_value_but_fixme_should_propagate_errors(); } }; diff --git a/Userland/Libraries/LibGUI/DynamicWidgetContainer.cpp b/Userland/Libraries/LibGUI/DynamicWidgetContainer.cpp index e2d7f86ea12c75..aa931c556ca333 100644 --- a/Userland/Libraries/LibGUI/DynamicWidgetContainer.cpp +++ b/Userland/Libraries/LibGUI/DynamicWidgetContainer.cpp @@ -241,13 +241,13 @@ ErrorOr DynamicWidgetContainer::detach_widgets() root_container->set_fill_with_background_color(true); root_container->set_layout(); root_container->set_frame_style(Gfx::FrameStyle::Window); - auto transfer_children = [this](auto reciever, auto children) { + auto transfer_children = [this](auto receiver, auto children) { for (NonnullRefPtr widget : children) { if (widget == m_controls_widget) continue; widget->remove_from_parent(); widget->set_visible(true); - reciever->add_child(widget); + receiver->add_child(widget); } }; diff --git a/Userland/Libraries/LibGfx/CMYKBitmap.h b/Userland/Libraries/LibGfx/CMYKBitmap.h index 2417a9628207b4..94b26fc7946c8c 100644 --- a/Userland/Libraries/LibGfx/CMYKBitmap.h +++ b/Userland/Libraries/LibGfx/CMYKBitmap.h @@ -35,6 +35,7 @@ class CMYKBitmap : public RefCounted { [[nodiscard]] CMYK* begin(); [[nodiscard]] CMYK* end(); + [[nodiscard]] ReadonlyBytes data() const { return m_data; } [[nodiscard]] size_t data_size() const { return m_data.size(); } ErrorOr> to_low_quality_rgb() const; diff --git a/Userland/Libraries/LibGfx/ICC/Profile.cpp b/Userland/Libraries/LibGfx/ICC/Profile.cpp index 9666dc84753600..f48273759d63eb 100644 --- a/Userland/Libraries/LibGfx/ICC/Profile.cpp +++ b/Userland/Libraries/LibGfx/ICC/Profile.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022-2023, Nico Weber + * Copyright (c) 2022-2025, Nico Weber * * SPDX-License-Identifier: BSD-2-Clause */ @@ -1282,12 +1282,12 @@ ErrorOr Profile::to_pcs_a_to_b(TagData const& tag_data, ReadonlyBy switch (tag_data.type()) { case Lut16TagData::Type: { auto const& a_to_b = static_cast(tag_data); - result = TRY(a_to_b.evaluate(data_color_space(), connection_space(), color)); + result = TRY(a_to_b.evaluate_to_pcs(data_color_space(), connection_space(), color)); break; } case Lut8TagData::Type: { auto const& a_to_b = static_cast(tag_data); - result = TRY(a_to_b.evaluate(data_color_space(), connection_space(), color)); + result = TRY(a_to_b.evaluate_to_pcs(data_color_space(), connection_space(), color)); break; } case LutAToBTagData::Type: { @@ -1491,12 +1491,14 @@ static TagSignature backward_transform_tag_for_rendering_intent(RenderingIntent ErrorOr Profile::from_pcs_b_to_a(TagData const& tag_data, FloatVector3 const& pcs, Bytes out_bytes) const { switch (tag_data.type()) { - case Lut16TagData::Type: - // FIXME - return Error::from_string_literal("ICC::Profile::to_pcs: BToA*Tag handling for mft2 tags not yet implemented"); - case Lut8TagData::Type: - // FIXME - return Error::from_string_literal("ICC::Profile::to_pcs: BToA*Tag handling for mft1 tags not yet implemented"); + case Lut16TagData::Type: { + auto const& a_to_b = static_cast(tag_data); + return a_to_b.evaluate_from_pcs(connection_space(), data_color_space(), pcs, out_bytes); + } + case Lut8TagData::Type: { + auto const& a_to_b = static_cast(tag_data); + return a_to_b.evaluate_from_pcs(connection_space(), data_color_space(), pcs, out_bytes); + } case LutBToATagData::Type: { auto const& b_to_a = static_cast(tag_data); if (b_to_a.number_of_input_channels() != number_of_components_in_color_space(connection_space())) @@ -1513,12 +1515,17 @@ ErrorOr Profile::from_pcs_b_to_a(TagData const& tag_data, FloatVector3 con ErrorOr Profile::from_pcs(Profile const& source_profile, FloatVector3 pcs, Bytes color) const { - if (source_profile.connection_space() != connection_space()) { - if (source_profile.connection_space() == ColorSpace::PCSLAB) { + return from_pcs(source_profile.connection_space(), source_profile.pcs_illuminant(), pcs, color); +} + +ErrorOr Profile::from_pcs(ColorSpace source_connection_space, XYZ const& source_illuminant, FloatVector3 pcs, Bytes color) const +{ + if (source_connection_space != connection_space()) { + if (source_connection_space == ColorSpace::PCSLAB) { VERIFY(connection_space() == ColorSpace::PCSXYZ); - pcs = xyz_from_lab(pcs, source_profile.pcs_illuminant()); + pcs = xyz_from_lab(pcs, source_illuminant); } else { - VERIFY(source_profile.connection_space() == ColorSpace::PCSXYZ); + VERIFY(source_connection_space == ColorSpace::PCSXYZ); VERIFY(connection_space() == ColorSpace::PCSLAB); pcs = lab_from_xyz(pcs, pcs_illuminant()); } @@ -1757,6 +1764,43 @@ ErrorOr Profile::convert_cmyk_image(Bitmap& out, CMYKBitmap const& in, Pro return {}; } +ErrorOr Profile::convert_cmyk_image_to_cmyk_image(CMYKBitmap& bitmap, Profile const& source_profile) const +{ + for (auto& pixel : bitmap) { + u8 cmyk[] = { pixel.c, pixel.m, pixel.y, pixel.k }; + auto pcs = TRY(source_profile.to_pcs(cmyk)); + TRY(from_pcs(source_profile, pcs, cmyk)); + pixel = CMYK { cmyk[0], cmyk[1], cmyk[2], cmyk[3] }; + } + + return {}; +} + +ErrorOr Profile::convert_image_to_cmyk_image(CMYKBitmap& out, Bitmap const& in, Profile const& source_profile) const +{ + if (out.size() != in.size()) + return Error::from_string_literal("ICC::Profile::convert_image_to_cmyk_image: out and in must have the same dimensions"); + + // Might fail if `out` has a scale_factor() != 1. + if (out.data_size() != in.data_size()) + return Error::from_string_literal("ICC::Profile::convert_image_to_cmyk_image: out and in must have the same buffer size"); + + static_assert(sizeof(ARGB32) == sizeof(CMYK)); + CMYK* out_data = out.begin(); + ARGB32 const* in_data = in.begin(); + + for (size_t i = 0; i < in.data_size() / sizeof(CMYK); ++i) { + u8 rgb[] = { Color::from_argb(in_data[i]).red(), Color::from_argb(in_data[i]).green(), Color::from_argb(in_data[i]).blue() }; + auto pcs = TRY(source_profile.to_pcs(rgb)); + + u8 cmyk[4]; + TRY(from_pcs(source_profile, pcs, cmyk)); + out_data[i] = CMYK { cmyk[0], cmyk[1], cmyk[2], cmyk[3] }; + } + + return {}; +} + XYZ const& Profile::red_matrix_column() const { return xyz_data(redMatrixColumnTag); } XYZ const& Profile::green_matrix_column() const { return xyz_data(greenMatrixColumnTag); } XYZ const& Profile::blue_matrix_column() const { return xyz_data(blueMatrixColumnTag); } diff --git a/Userland/Libraries/LibGfx/ICC/Profile.h b/Userland/Libraries/LibGfx/ICC/Profile.h index 567f994c31e5d1..e9aed9539aa089 100644 --- a/Userland/Libraries/LibGfx/ICC/Profile.h +++ b/Userland/Libraries/LibGfx/ICC/Profile.h @@ -288,12 +288,16 @@ class Profile : public RefCounted { // Converts from the profile connection space to an 8-bits-per-channel color. // The notes on `to_pcs()` apply to this too. ErrorOr from_pcs(Profile const& source_profile, FloatVector3, Bytes) const; + ErrorOr from_pcs(ColorSpace source_connection_space, XYZ const& source_illuminant, FloatVector3, Bytes) const; ErrorOr to_lab(ReadonlyBytes) const; ErrorOr convert_image(Bitmap&, Profile const& source_profile) const; ErrorOr convert_cmyk_image(Bitmap&, CMYKBitmap const&, Profile const& source_profile) const; + ErrorOr convert_cmyk_image_to_cmyk_image(CMYKBitmap&, Profile const& source_profile) const; + ErrorOr convert_image_to_cmyk_image(CMYKBitmap&, Bitmap const&, Profile const& source_profile) const; + // Only call these if you know that this is an RGB matrix-based profile. XYZ const& red_matrix_column() const; XYZ const& green_matrix_column() const; diff --git a/Userland/Libraries/LibGfx/ICC/TagTypes.h b/Userland/Libraries/LibGfx/ICC/TagTypes.h index 92b804077244f5..ff525d389fdb86 100644 --- a/Userland/Libraries/LibGfx/ICC/TagTypes.h +++ b/Userland/Libraries/LibGfx/ICC/TagTypes.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2023, Nico Weber + * Copyright (c) 2023-2025, Nico Weber * * SPDX-License-Identifier: BSD-2-Clause */ @@ -63,6 +63,33 @@ inline FloatVector3 lerp_nd(Function size, FunctionND instead of ND->3D. +inline void lerp_nd(IntVector3 size, Function)> sample, FloatVector3 const& x, Span scratch, Span out) +{ + unsigned left_index[3]; + float factor[3]; + for (size_t i = 0; i < 3; ++i) { + unsigned n = size[i] - 1; + float ec = x[i] * n; + left_index[i] = min(static_cast(ec), n - 1); + factor[i] = ec - left_index[i]; + } + + out.fill(0.0f); + // The i'th bit of mask indicates if the i'th coordinate is rounded up or down. + IntVector3 coordinates; + for (size_t mask = 0; mask < (1u << 3); ++mask) { + float sample_weight = 1.0f; + for (size_t i = 0; i < 3; ++i) { + coordinates[i] = left_index[i] + ((mask >> i) & 1u); + sample_weight *= ((mask >> i) & 1u) ? factor[i] : 1.0f - factor[i]; + } + sample(coordinates, scratch); + for (size_t i = 0; i < out.size(); ++i) + out[i] += scratch[i] * sample_weight; + } +} + using S15Fixed16 = FixedPoint<16, i32>; using U16Fixed16 = FixedPoint<16, u32>; @@ -317,7 +344,10 @@ class Lut16TagData : public TagData { Vector const& clut_values() const { return m_clut_values; } Vector const& output_tables() const { return m_output_tables; } - ErrorOr evaluate(ColorSpace input_space, ColorSpace connection_space, ReadonlyBytes) const; + // FIXME: If we add DeviceLink support, this can become an arbitrary nD -> nD transform. + // For now, 3D -> nD and nD -> 3D is sufficient. + ErrorOr evaluate_to_pcs(ColorSpace input_space, ColorSpace connection_space, ReadonlyBytes) const; + ErrorOr evaluate_from_pcs(ColorSpace connection_space, ColorSpace output_space, FloatVector3, Bytes) const; private: EMatrix3x3 m_e; @@ -376,7 +406,10 @@ class Lut8TagData : public TagData { Vector const& clut_values() const { return m_clut_values; } Vector const& output_tables() const { return m_output_tables; } - ErrorOr evaluate(ColorSpace input_space, ColorSpace connection_space, ReadonlyBytes) const; + // FIXME: If we add DeviceLink support, this can become an arbitrary nD -> nD transform. + // For now, 3D -> nD and nD -> 3D is sufficient. + ErrorOr evaluate_to_pcs(ColorSpace input_space, ColorSpace connection_space, ReadonlyBytes) const; + ErrorOr evaluate_from_pcs(ColorSpace connection_space, ColorSpace output_space, FloatVector3, Bytes) const; private: EMatrix3x3 m_e; @@ -1035,24 +1068,22 @@ class XYZTagData : public TagData { Vector m_xyzs; }; -inline ErrorOr Lut16TagData::evaluate(ColorSpace input_space, ColorSpace connection_space, ReadonlyBytes color_u8) const +inline ErrorOr Lut16TagData::evaluate_to_pcs(ColorSpace input_space, ColorSpace connection_space, ReadonlyBytes color_u8) const { // See comment at start of LutAToBTagData::evaluate() for the clipping flow. VERIFY(connection_space == ColorSpace::PCSXYZ || connection_space == ColorSpace::PCSLAB); VERIFY(number_of_input_channels() == color_u8.size()); - - // FIXME: This will be wrong once Profile::from_pcs_b_to_a() calls this function too. VERIFY(number_of_output_channels() == 3); - // ICC v4, 10.11 lut8Type + // ICC v4, 10.10 lut16Type // "Data is processed using these elements via the following sequence: - // (matrix) ⇨ (1d input tables) ⇨ (multi-dimensional lookup table, CLUT) ⇨ (1d output tables)" + // (matrix) ⇨ (1D input tables) ⇨ (multi-dimensional lookup table, CLUT) ⇨ (1D output tables)" Vector color; for (u8 c : color_u8) color.append(c / 255.0f); - // "3 x 3 matrix (which shall be the identity matrix unless the input colour space is PCSXYZ)" + // "The matrix shall be an identity matrix unless the input is in the PCSXYZ colour space." // In practice, it's usually RGB or CMYK. if (input_space == ColorSpace::PCSXYZ) { EMatrix3x3 const& e = m_e; @@ -1112,10 +1143,12 @@ inline ErrorOr Lut16TagData::evaluate(ColorSpace input_space, Colo // shall be clipped on a per-component basis." output_color *= 65535.0f / 65280.0f; - // Table 42 — Legacy PCSLAB L* encoding + // Table 12 — PCSLAB L* encoding + // (The multiplication above and using Table 12 is equivalent to using Table 42 — Legacy PCSLAB L* encoding.) output_color[0] = clamp(output_color[0] * 100.0f, 0.0f, 100.0f); - // Table 43 — Legacy PCSLAB a* or PCSLAB b* encoding + // Table 13 — PCSLAB a* or PCSLAB b* encoding + // (The multiplication above and using Table 12 is equivalent to using Table 43 — Legacy PCSLAB a* or PCSLAB b* encoding.) output_color[1] = clamp(output_color[1] * 255.0f - 128.0f, -128.0f, 127.0f); output_color[2] = clamp(output_color[2] * 255.0f - 128.0f, -128.0f, 127.0f); } @@ -1123,13 +1156,103 @@ inline ErrorOr Lut16TagData::evaluate(ColorSpace input_space, Colo return output_color; } -inline ErrorOr Lut8TagData::evaluate(ColorSpace input_space, ColorSpace connection_space, ReadonlyBytes color_u8) const +inline ErrorOr Lut16TagData::evaluate_from_pcs(ColorSpace connection_space, ColorSpace output_space, FloatVector3 pcs, Bytes color_u8) const +{ + // This is very similar to Lut16TagData::evaluate_from_pcs(), but instead of converting from device space to PCS, + // it converts from PCS to device space. + VERIFY(connection_space == ColorSpace::PCSXYZ || connection_space == ColorSpace::PCSLAB); + VERIFY(number_of_input_channels() == 3); + VERIFY(number_of_output_channels() == color_u8.size()); + + // ICC v4, 10.10 lut16Type + // "Data is processed using these elements via the following sequence: + // (matrix) ⇨ (1D input tables) ⇨ (multi-dimensional lookup table, CLUT) ⇨ (1D output tables)" + + if (connection_space == ColorSpace::PCSXYZ) { + // Table 11 - PCSXYZ X, Y or Z encoding + pcs *= 32768.0f / 65535; + } else { + VERIFY(connection_space == ColorSpace::PCSLAB); + + // ICC v4, 6.3.4.2 General PCS encoding + // Table 12 — PCSLAB L* encoding + pcs[0] = clamp(pcs[0] / 100.0f, 0.0f, 1.0f); + + // Table 13 — PCSLAB a* or PCSLAB b* encoding + pcs[1] = clamp((pcs[1] + 128.0f) / 255.0f, 0.0f, 1.0f); + pcs[2] = clamp((pcs[2] + 128.0f) / 255.0f, 0.0f, 1.0f); + + pcs *= 65280.0f / 65535; + } + + // "3 x 3 matrix (which shall be the identity matrix unless the input colour space is PCSXYZ)" + if (connection_space == ColorSpace::PCSXYZ) { + EMatrix3x3 const& e = m_e; + pcs = FloatVector3 { + (float)e[0] * pcs[0] + (float)e[1] * pcs[1] + (float)e[2] * pcs[2], + (float)e[3] * pcs[0] + (float)e[4] * pcs[1] + (float)e[5] * pcs[2], + (float)e[6] * pcs[0] + (float)e[7] * pcs[1] + (float)e[8] * pcs[2], + }; + } + + // "The input tables are arrays of 16-bit unsigned values. Each input table consists of a minimum of two and a maximum of 4096 uInt16Number integers. + // Each input table entry is appropriately normalized to the range 0 to 65535. + // The inputTable is of size (InputChannels x inputTableEntries x 2) bytes. + // When stored in this tag, the one-dimensional lookup tables are packed one after another" + for (size_t c = 0; c < 3; ++c) + pcs[c] = lerp_1d(m_input_tables.span().slice(c * m_number_of_input_table_entries, m_number_of_input_table_entries), pcs[c]) / 65535.0f; + + // "The CLUT is organized as an i-dimensional array with a given number of grid points in each dimension, + // where i is the number of input channels (input tables) in the transform. + // The dimension corresponding to the first input channel varies least rapidly and + // the dimension corresponding to the last input channel varies most rapidly. + // Each grid point value is an o-byte array, where o is the number of output channels. + // The first sequential byte of the entry contains the function value for the first output function, + // the second sequential byte of the entry contains the function value for the second output function, + // and so on until all the output functions have been supplied." + auto sample = [this](IntVector3 const& coordinates, Span out) { + size_t stride = out.size(); + size_t offset = 0; + for (int i = 3 - 1; i >= 0; --i) { + offset += coordinates[i] * stride; + stride *= m_number_of_clut_grid_points; + } + for (size_t c = 0; c < out.size(); ++c) + out[c] = (float)m_clut_values[offset + c]; + }; + + Vector scratch; + Vector color; + scratch.resize(number_of_output_channels()); + color.resize(number_of_output_channels()); + + lerp_nd({ m_number_of_clut_grid_points, m_number_of_clut_grid_points, m_number_of_clut_grid_points }, move(sample), pcs, scratch.span(), color.span()); + + // "The output tables are arrays of 16-bit unsigned values. Each output table consists of a minimum of two and a maximum of 4096 uInt16Number integers. + // Each output table entry is appropriately normalized to the range 0 to 65535. + // The outputTable is of size (OutputChannels x outputTableEntries x 2) bytes. + // When stored in this tag, the one-dimensional lookup tables are packed one after another" + for (u8 c = 0; c < color.size(); ++c) + color[c] = lerp_1d(m_output_tables.span().slice(c * m_number_of_output_table_entries, m_number_of_output_table_entries), color[c] / 65535.0f) / 65535.0f; + + // Since the LUTs assume that everything's in 0..1 and we assume that's mapped linearly to bytes, + // we don't need to look at output_space. + // 6.5 Device encoding + // "The specification of device value encoding is determined by the device. Normally, device values in the range of + // 0,0 to 1,0 are encoded using a 0 to 255 (FFh) range when using 8 bits" + (void)output_space; + + for (u8 c = 0; c < color_u8.size(); ++c) + color_u8[c] = round_to(clamp(color[c] * 255.0f, 0.0f, 255.0f)); + + return {}; +} + +inline ErrorOr Lut8TagData::evaluate_to_pcs(ColorSpace input_space, ColorSpace connection_space, ReadonlyBytes color_u8) const { // See comment at start of LutAToBTagData::evaluate() for the clipping flow. VERIFY(connection_space == ColorSpace::PCSXYZ || connection_space == ColorSpace::PCSLAB); VERIFY(number_of_input_channels() == color_u8.size()); - - // FIXME: This will be wrong once Profile::from_pcs_b_to_a() calls this function too. VERIFY(number_of_output_channels() == 3); // ICC v4, 10.11 lut8Type @@ -1202,6 +1325,96 @@ inline ErrorOr Lut8TagData::evaluate(ColorSpace input_space, Color return output_color; } +inline ErrorOr Lut8TagData::evaluate_from_pcs(ColorSpace connection_space, ColorSpace output_space, FloatVector3 pcs, Bytes color_u8) const +{ + // This is very similar to Lut8TagData::evaluate_from_pcs(), but instead of converting from device space to PCS, + // it converts from PCS to device space. + VERIFY(connection_space == ColorSpace::PCSXYZ || connection_space == ColorSpace::PCSLAB); + VERIFY(number_of_input_channels() == 3); + VERIFY(number_of_output_channels() == color_u8.size()); + + // ICC v4, 10.11 lut8Type + // "Data is processed using these elements via the following sequence: + // (matrix) ⇨ (1d input tables) ⇨ (multi-dimensional lookup table, CLUT) ⇨ (1d output tables)" + + if (connection_space == ColorSpace::PCSXYZ) { + // "An 8-bit PCSXYZ encoding has not been defined, so the interpretation of a lut8Type in a profile that uses PCSXYZ is implementation specific." + } else { + VERIFY(connection_space == ColorSpace::PCSLAB); + + // ICC v4, 6.3.4.2 General PCS encoding + // Table 12 — PCSLAB L* encoding + pcs[0] = clamp(pcs[0] / 100.0f, 0.0f, 1.0f); + + // Table 13 — PCSLAB a* or PCSLAB b* encoding + pcs[1] = clamp((pcs[1] + 128.0f) / 255.0f, 0.0f, 1.0f); + pcs[2] = clamp((pcs[2] + 128.0f) / 255.0f, 0.0f, 1.0f); + } + + // "3 x 3 matrix (which shall be the identity matrix unless the input colour space is PCSXYZ)" + // Since "An 8-bit PCSXYZ encoding has not been defined", this should never happen in practice. + if (connection_space == ColorSpace::PCSXYZ) { + EMatrix3x3 const& e = m_e; + pcs = FloatVector3 { + (float)e[0] * pcs[0] + (float)e[1] * pcs[1] + (float)e[2] * pcs[2], + (float)e[3] * pcs[0] + (float)e[4] * pcs[1] + (float)e[5] * pcs[2], + (float)e[6] * pcs[0] + (float)e[7] * pcs[1] + (float)e[8] * pcs[2], + }; + } + + // "The input tables are arrays of uInt8Number values. Each input table consists of 256 uInt8Number integers. + // Each input table entry is appropriately normalized to the range 0 to 255. + // The inputTable is of size (InputChannels x 256) bytes. + // When stored in this tag, the one-dimensional lookup tables are packed one after another" + for (size_t c = 0; c < 3; ++c) + pcs[c] = lerp_1d(m_input_tables.span().slice(c * 256, 256), pcs[c]) / 255.0f; + + // "The CLUT is organized as an i-dimensional array with a given number of grid points in each dimension, + // where i is the number of input channels (input tables) in the transform. + // The dimension corresponding to the first input channel varies least rapidly and + // the dimension corresponding to the last input channel varies most rapidly. + // Each grid point value is an o-byte array, where o is the number of output channels. + // The first sequential byte of the entry contains the function value for the first output function, + // the second sequential byte of the entry contains the function value for the second output function, + // and so on until all the output functions have been supplied." + auto sample = [this](IntVector3 const& coordinates, Span out) { + size_t stride = out.size(); + size_t offset = 0; + for (int i = 3 - 1; i >= 0; --i) { + offset += coordinates[i] * stride; + stride *= m_number_of_clut_grid_points; + } + for (size_t c = 0; c < out.size(); ++c) + out[c] = (float)m_clut_values[offset + c]; + }; + + Vector scratch; + Vector color; + scratch.resize(number_of_output_channels()); + color.resize(number_of_output_channels()); + + lerp_nd({ m_number_of_clut_grid_points, m_number_of_clut_grid_points, m_number_of_clut_grid_points }, move(sample), pcs, scratch.span(), color.span()); + + // "The output tables are arrays of uInt8Number values. Each output table consists of 256 uInt8Number integers. + // Each output table entry is appropriately normalized to the range 0 to 255. + // The outputTable is of size (OutputChannels x 256) bytes. + // When stored in this tag, the one-dimensional lookup tables are packed one after another" + for (u8 c = 0; c < color.size(); ++c) + color[c] = lerp_1d(m_output_tables.span().slice(c * 256, 256), color[c] / 255.0f) / 255.0f; + + // Since the LUTs assume that everything's in 0..1 and we assume that's mapped linearly to bytes, + // we don't need to look at output_space. + // 6.5 Device encoding + // "The specification of device value encoding is determined by the device. Normally, device values in the range of + // 0,0 to 1,0 are encoded using a 0 to 255 (FFh) range when using 8 bits" + (void)output_space; + + for (u8 c = 0; c < color_u8.size(); ++c) + color_u8[c] = round_to(clamp(color[c] * 255.0f, 0.0f, 255.0f)); + + return {}; +} + inline ErrorOr LutAToBTagData::evaluate(ColorSpace connection_space, ReadonlyBytes color_u8) const { VERIFY(connection_space == ColorSpace::PCSXYZ || connection_space == ColorSpace::PCSLAB); @@ -1330,6 +1543,7 @@ inline ErrorOr LutBToATagData::evaluate(ColorSpace connection_space, Float color[1] = (in_color[1] + 128.0f) / 255.0f; color[2] = (in_color[2] + 128.0f) / 255.0f; } + color = color.clamped(0.f, 1.f); color = FloatVector3 { evaluate_curve(m_b_curves[0], color[0]), @@ -1359,8 +1573,50 @@ inline ErrorOr LutBToATagData::evaluate(ColorSpace connection_space, Float VERIFY(m_clut.has_value() == m_a_curves.has_value()); if (m_clut.has_value()) { - // FIXME - return Error::from_string_literal("LutBToATagData::evaluate: Not yet implemented when CLUT present"); + Vector scratch; + Vector out; + scratch.resize(number_of_output_channels()); + out.resize(number_of_output_channels()); + + auto const& clut = m_clut.value(); + auto sample1 = [&clut](Vector const& data, IntVector3 const& coordinates, Span out) { + size_t stride = out.size(); + size_t offset = 0; + for (int i = 3 - 1; i >= 0; --i) { + offset += coordinates[i] * stride; + stride *= clut.number_of_grid_points_in_dimension[i]; + } + for (size_t c = 0; c < out.size(); ++c) + out[c] = (float)data[offset + c]; + }; + auto sample = [&clut, &sample1](IntVector3 const& coordinates, Span out) { + clut.values.visit( + [&](Vector const& v) { + sample1(v, coordinates, out); + for (auto& f : out) + f /= 255.0f; + }, + [&](Vector const& v) { + sample1(v, coordinates, out); + for (auto& f : out) + f /= 65535.0f; + }); + }; + + VERIFY(clut.number_of_grid_points_in_dimension.size() == 3); + Gfx::IntVector3 size { + clut.number_of_grid_points_in_dimension[0], + clut.number_of_grid_points_in_dimension[1], + clut.number_of_grid_points_in_dimension[2], + }; + lerp_nd(size, move(sample), color, scratch, out); + + auto const& a_curves = m_a_curves.value(); + VERIFY(a_curves.size() == out.size()); + for (unsigned c = 0; c < out.size(); ++c) { + out[c] = evaluate_curve(a_curves[c], out[c]); + out_bytes[c] = round_to(clamp(out[c] * 255.0f, 0.0f, 255.0f)); + } } else { VERIFY(number_of_output_channels() == 3); out_bytes[0] = round_to(color[0] * 255.0f); diff --git a/Userland/Libraries/LibGfx/ICC/WellKnownProfiles.cpp b/Userland/Libraries/LibGfx/ICC/WellKnownProfiles.cpp index e7f23fe9478f4e..2f14992b394575 100644 --- a/Userland/Libraries/LibGfx/ICC/WellKnownProfiles.cpp +++ b/Userland/Libraries/LibGfx/ICC/WellKnownProfiles.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2023, Nico Weber + * Copyright (c) 2023-2025, Nico Weber * * SPDX-License-Identifier: BSD-2-Clause */ @@ -11,13 +11,13 @@ namespace Gfx::ICC { -static ProfileHeader rgb_header() +static ProfileHeader profile_header(ColorSpace data_color_space, ColorSpace connection_space) { ProfileHeader header; header.version = Version(4, 0x40); header.device_class = DeviceClass::DisplayDevice; - header.data_color_space = ColorSpace::RGB; - header.connection_space = ColorSpace::PCSXYZ; + header.data_color_space = data_color_space; + header.connection_space = connection_space; header.creation_timestamp = MUST(DateTime::from_time_t(0)); header.rendering_intent = RenderingIntent::Perceptual; header.pcs_illuminant = XYZ { 0.9642, 1.0, 0.8249 }; @@ -38,6 +38,273 @@ static ErrorOr> XYZ_data(XYZ xyz) return try_make_ref_counted(0, 0, move(xyzs)); } +static EMatrix3x3 identity_matrix() +{ + return EMatrix3x3 { + S15Fixed16(1.0), + S15Fixed16(0.0), + S15Fixed16(0.0), + S15Fixed16(0.0), + S15Fixed16(1.0), + S15Fixed16(0.0), + S15Fixed16(0.0), + S15Fixed16(0.0), + S15Fixed16(1.0), + }; +} + +template +Vector make_2x2x2_cube() +{ + Vector values; + auto add_entry = [&values](T x, T y, T z) { + values.append(x); + values.append(y); + values.append(z); + }; + auto const N = NumericLimits::max(); + add_entry(0, 0, 0); + add_entry(0, 0, N); + add_entry(0, N, 0); + add_entry(0, N, N); + add_entry(N, 0, 0); + add_entry(N, 0, N); + add_entry(N, N, 0); + add_entry(N, N, N); + return values; +} + +template +CLUTData make_2x2x2_cube_clut() +{ + Vector number_of_grid_points_in_dimension; + for (int i = 0; i < 3; ++i) + number_of_grid_points_in_dimension.append(2); + return CLUTData { + move(number_of_grid_points_in_dimension), + make_2x2x2_cube() + }; +} + +ErrorOr> IdentityLAB() +{ + // Identity mapping between CIELAB and PCSLAB. + + auto header = profile_header(ColorSpace::CIELAB, ColorSpace::PCSLAB); + header.device_class = DeviceClass::ColorSpace; + + OrderedHashMap> tag_table; + + TRY(tag_table.try_set(profileDescriptionTag, TRY(en_US("SerenityOS Identity LAB"sv)))); + TRY(tag_table.try_set(copyrightTag, TRY(en_US("Public Domain"sv)))); + + // Use an identity 8-bit mft1 lookup table. + auto e_matrix = identity_matrix(); + + Vector input_tables; + for (int c = 0; c < 3; ++c) { + // mft1 requires 256 entries. + for (int i = 0; i < 256; ++i) + input_tables.append(i); + } + + auto clut_values = make_2x2x2_cube(); + Vector output_tables = input_tables; + + auto mft1 = TRY(try_make_ref_counted(0, 0, e_matrix, + 3, 3, 2, + 256, 256, + move(input_tables), move(clut_values), move(output_tables))); + TRY(tag_table.try_set(AToB0Tag, mft1)); + TRY(tag_table.try_set(BToA0Tag, mft1)); + + // White point. + TRY(tag_table.try_set(mediaWhitePointTag, TRY(XYZ_data(header.pcs_illuminant)))); + + return Profile::create(header, move(tag_table)); +} + +ErrorOr> IdentityLAB_mft2() +{ + // Identity mapping between CIELAB and PCSXYZ, using an unnecessarily large mft2. + + auto header = profile_header(ColorSpace::CIELAB, ColorSpace::PCSLAB); + header.device_class = DeviceClass::ColorSpace; + + OrderedHashMap> tag_table; + + TRY(tag_table.try_set(profileDescriptionTag, TRY(en_US("SerenityOS Identity LAB, mft2"sv)))); + TRY(tag_table.try_set(copyrightTag, TRY(en_US("Public Domain"sv)))); + + // mft1 is plenty; this is just for testing mft2 LAB codepaths. + EMatrix3x3 e_matrix = identity_matrix(); + + Vector input_tables; + for (int c = 0; c < 3; ++c) { + // mft2 allows between 2 and 4096 entries. If there are just two values, it linearly maps between them. + input_tables.append(0); + input_tables.append(65535); + } + + auto clut_values = make_2x2x2_cube(); + Vector output_tables = input_tables; + + auto mft2 = TRY(try_make_ref_counted(0, 0, e_matrix, + 3, 3, 2, + 2, 2, + move(input_tables), move(clut_values), move(output_tables))); + TRY(tag_table.try_set(AToB0Tag, mft2)); + TRY(tag_table.try_set(BToA0Tag, mft2)); + + // White point. + TRY(tag_table.try_set(mediaWhitePointTag, TRY(XYZ_data(header.pcs_illuminant)))); + + return Profile::create(header, move(tag_table)); +} + +ErrorOr> IdentityXYZ_D50() +{ + // Identity mapping between nCIEXYZ and PCSXYZ. + + auto header = profile_header(ColorSpace::nCIEXYZ, ColorSpace::PCSXYZ); + header.device_class = DeviceClass::ColorSpace; + + OrderedHashMap> tag_table; + + TRY(tag_table.try_set(profileDescriptionTag, TRY(en_US("SerenityOS Identity XYZ"sv)))); + TRY(tag_table.try_set(copyrightTag, TRY(en_US("Public Domain"sv)))); + + // "An 8-bit PCSXYZ encoding has not been defined", so use an identity 16-bit mft2 lookup table. + EMatrix3x3 e_matrix = identity_matrix(); + + Vector input_tables; + for (int c = 0; c < 3; ++c) { + // mft2 allows between 2 and 4096 entries. If there are just two values, it linearly maps between them. + input_tables.append(0); + input_tables.append(65535); + } + + auto clut_values = make_2x2x2_cube(); + Vector output_tables = input_tables; + + auto mft2 = TRY(try_make_ref_counted(0, 0, e_matrix, + 3, 3, 2, + 2, 2, + move(input_tables), move(clut_values), move(output_tables))); + TRY(tag_table.try_set(AToB0Tag, mft2)); + TRY(tag_table.try_set(BToA0Tag, mft2)); + + // White point. + TRY(tag_table.try_set(mediaWhitePointTag, TRY(XYZ_data(header.pcs_illuminant)))); + + return Profile::create(header, move(tag_table)); +} + +static EMatrix3x4 identity_matrix_3x4() +{ + return EMatrix3x4 { + S15Fixed16(1.0), + S15Fixed16(0.0), + S15Fixed16(0.0), + S15Fixed16(0.0), + S15Fixed16(1.0), + S15Fixed16(0.0), + S15Fixed16(0.0), + S15Fixed16(0.0), + S15Fixed16(1.0), + + S15Fixed16(0.0), + S15Fixed16(0.0), + S15Fixed16(0.0), + }; +} + +static ErrorOr> identity_curve() +{ + Array curve_parameters = { 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0 }; + return try_make_ref_counted(0, 0, ParametricCurveTagData::FunctionType::Type0, curve_parameters); +} + +static ErrorOr> make_identity_curves() +{ + Vector identity_curves; + auto linear = TRY(identity_curve()); + for (int c = 0; c < 3; ++c) + TRY(identity_curves.try_append(linear)); + return identity_curves; +} + +static ErrorOr> Identity_mABmBA(StringView description, ColorSpace data_color_space, ColorSpace connection_space, Optional clut) +{ + auto header = profile_header(data_color_space, connection_space); + header.device_class = DeviceClass::ColorSpace; + + OrderedHashMap> tag_table; + + TRY(tag_table.try_set(profileDescriptionTag, TRY(en_US(description)))); + TRY(tag_table.try_set(copyrightTag, TRY(en_US("Public Domain"sv)))); + + Vector identity_curves = TRY(make_identity_curves()); + + Optional> a_curves; + if (clut.has_value()) + a_curves = identity_curves; + + Optional> m_curves; + Optional e_matrix; + + // FIXME: Could make these two optional too. + m_curves = identity_curves; + e_matrix = identity_matrix_3x4(); + + Vector b_curves = identity_curves; + + auto a_to_b = TRY(try_make_ref_counted(0, 0, + 3, 3, + a_curves, clut, m_curves, e_matrix, b_curves)); + TRY(tag_table.try_set(AToB0Tag, a_to_b)); + + auto b_to_a = TRY(try_make_ref_counted(0, 0, + 3, 3, + move(b_curves), e_matrix, move(m_curves), move(clut), move(a_curves))); + TRY(tag_table.try_set(BToA0Tag, b_to_a)); + + // White point. + TRY(tag_table.try_set(mediaWhitePointTag, TRY(XYZ_data(header.pcs_illuminant)))); + + return Profile::create(header, move(tag_table)); +} + +ErrorOr> IdentityLAB_mABmBA_no_clut() +{ + // Identity mapping between CIELAB and PCSLAB, using mAB / mBA tags. + return Identity_mABmBA("SerenityOS Identity LAB, mAB/mBA, no CLUT"sv, ColorSpace::CIELAB, ColorSpace::PCSLAB, {}); +} + +ErrorOr> IdentityLAB_mABmBA_u8_clut() +{ + // Identity mapping between CIELAB and PCSLAB, using mAB / mBA tags. + return Identity_mABmBA("SerenityOS Identity LAB, mAB/mBA, u8 CLUT"sv, ColorSpace::CIELAB, ColorSpace::PCSLAB, make_2x2x2_cube_clut()); +} + +ErrorOr> IdentityLAB_mABmBA_u16_clut() +{ + // Identity mapping between CIELAB and PCSLAB, using mAB / mBA tags. + return Identity_mABmBA("SerenityOS Identity LAB, mAB/mBA, u16 CLUT"sv, ColorSpace::CIELAB, ColorSpace::PCSLAB, make_2x2x2_cube_clut()); +} + +ErrorOr> IdentityXYZ_D50_mABmBA_no_clut() +{ + // Identity mapping between CIELAB and PCSLAB, using mAB / mBA tags. + return Identity_mABmBA("SerenityOS Identity XYZ, mAB/mBA, no CLUT"sv, ColorSpace::nCIEXYZ, ColorSpace::PCSXYZ, {}); +} + +ErrorOr> IdentityXYZ_D50_mABmBA_u16_clut() +{ + // Identity mapping between CIELAB and PCSLAB, using mAB / mBA tags. + return Identity_mABmBA("SerenityOS Identity XYZ, mAB/mBA, u16 CLUT"sv, ColorSpace::nCIEXYZ, ColorSpace::PCSXYZ, make_2x2x2_cube_clut()); +} + ErrorOr> sRGB_curve() { // Numbers from https://en.wikipedia.org/wiki/SRGB#From_sRGB_to_CIE_XYZ @@ -54,7 +321,7 @@ ErrorOr> sRGB() // Explain why, and why this picks the numbers it does. // In the meantime, https://github.com/SerenityOS/serenity/pull/17714 has a few notes. - auto header = rgb_header(); + auto header = profile_header(ColorSpace::RGB, ColorSpace::PCSXYZ); OrderedHashMap> tag_table; diff --git a/Userland/Libraries/LibGfx/ICC/WellKnownProfiles.h b/Userland/Libraries/LibGfx/ICC/WellKnownProfiles.h index 2ff2ca3c4291e6..57762d33422c7f 100644 --- a/Userland/Libraries/LibGfx/ICC/WellKnownProfiles.h +++ b/Userland/Libraries/LibGfx/ICC/WellKnownProfiles.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2023, Nico Weber + * Copyright (c) 2023-2025, Nico Weber * * SPDX-License-Identifier: BSD-2-Clause */ @@ -14,6 +14,16 @@ namespace Gfx::ICC { class Profile; class ParametricCurveTagData; +ErrorOr> IdentityLAB(); +ErrorOr> IdentityLAB_mft2(); +ErrorOr> IdentityLAB_mABmBA_no_clut(); +ErrorOr> IdentityLAB_mABmBA_u8_clut(); +ErrorOr> IdentityLAB_mABmBA_u16_clut(); + +ErrorOr> IdentityXYZ_D50(); +ErrorOr> IdentityXYZ_D50_mABmBA_no_clut(); +ErrorOr> IdentityXYZ_D50_mABmBA_u16_clut(); + ErrorOr> sRGB(); ErrorOr> sRGB_curve(); diff --git a/Userland/Libraries/LibGfx/ImageFormats/BilevelImage.cpp b/Userland/Libraries/LibGfx/ImageFormats/BilevelImage.cpp index 61c2a62c51956d..b23c605f920eff 100644 --- a/Userland/Libraries/LibGfx/ImageFormats/BilevelImage.cpp +++ b/Userland/Libraries/LibGfx/ImageFormats/BilevelImage.cpp @@ -246,6 +246,24 @@ static constexpr auto bayer_matrix_8x8 = make_bayer_matrix<3>(); ErrorOr> BilevelImage::create_from_bitmap(Gfx::Bitmap const& bitmap, DitheringAlgorithm dithering_algorithm) { + bool input_is_bilevel = true; + for (auto pixel : bitmap) { + if (pixel != 0xFF000000 && pixel != 0xFFFFFFFF) { + input_is_bilevel = false; + break; + } + } + if (input_is_bilevel) { + auto bilevel_image = TRY(BilevelImage::create(bitmap.width(), bitmap.height())); + for (int y = 0; y < bitmap.height(); ++y) { + for (int x = 0; x < bitmap.width(); ++x) { + auto color = bitmap.get_pixel(x, y); + bilevel_image->set_bit(x, y, color == Color::Black ? 1 : 0); + } + } + return bilevel_image; + } + auto gray_bitmap = TRY(ByteBuffer::create_uninitialized(bitmap.width() * bitmap.height())); for (int y = 0, i = 0; y < bitmap.height(); ++y) { for (int x = 0; x < bitmap.width(); ++x, ++i) { @@ -294,9 +312,13 @@ ErrorOr> BilevelImage::create_from_bitmap(Gfx::Bitma VERIFY(is_power_of_two(n)); auto mask = n - 1; + // A bayer matrix of dimension N has N x N +1 different states. First one + // is an all black matrix, and then one more for each element turning white. + u32 number_of_states = n * n + 1; + for (int y = 0, i = 0; y < bitmap.height(); ++y) { for (int x = 0; x < bitmap.width(); ++x, ++i) { - u8 threshold = (bayer_matrix[(y & mask) * n + (x & mask)] * 255) / ((n * n) - 1); + u8 threshold = round_to((1 + bayer_matrix[(y & mask) * n + (x & mask)]) * 255.f / number_of_states); bilevel_image->set_bit(x, y, gray_bitmap[i] > threshold ? 0 : 1); } } diff --git a/Userland/Libraries/LibGfx/ImageFormats/JBIG2Loader.cpp b/Userland/Libraries/LibGfx/ImageFormats/JBIG2Loader.cpp index d26523cceb1374..de473b67793737 100644 --- a/Userland/Libraries/LibGfx/ImageFormats/JBIG2Loader.cpp +++ b/Userland/Libraries/LibGfx/ImageFormats/JBIG2Loader.cpp @@ -141,6 +141,18 @@ struct SegmentData { // Set on dictionary segments after they've been decoded. Optional> symbols; + struct BitmapCodingContextState { + Optional generic_contexts; + Optional refinement_contexts; + bool used_huffman_encoding { false }; + bool used_refinement_or_aggregate_coding { false }; + u8 symbol_template { 0 }; + u8 refinement_template { 0 }; + Array adaptive_template_pixels {}; + Array refinement_adaptive_template_pixels {}; + }; + Optional retained_bitmap_coding_contexts; // Only set on dictionary segments with bitmap_coding_context_retained set. + // Set on pattern segments after they've been decoded. Optional> patterns; @@ -185,7 +197,7 @@ struct JBIG2LoadingContext { // Files from the Power JBIG2 tests have a few quirks. // Since they're useful for coverage, detect these files and be more lenient. - bool is_power_jbig2_file { false }; + bool allow_power_jbig2_quirks { false }; }; static ErrorOr decode_jbig2_header(JBIG2LoadingContext& context, ReadonlyBytes data) @@ -384,6 +396,9 @@ static ErrorOr scan_for_immediate_generic_region_size(ReadonlyBytes data static void identify_power_jbig2_files(JBIG2LoadingContext& context) { + if (context.options.strictness == JBIG2DecoderOptions::Strictness::SpecCompliant) + return; + for (auto const& segment : context.segments) { auto signature_data_1 = "\x20\0\0\0" "Source\0" @@ -398,7 +413,7 @@ static void identify_power_jbig2_files(JBIG2LoadingContext& context) "1.0.0\0" "\0"sv; if (segment.type() == JBIG2::SegmentType::Extension && (segment.data == signature_data_1.bytes() || segment.data == signature_data_2.bytes())) { - context.is_power_jbig2_file = true; + context.allow_power_jbig2_quirks = true; return; } } @@ -438,18 +453,29 @@ static ErrorOr validate_segment_header_retention_flags(JBIG2LoadingContext return Error::from_string_literal("JBIG2ImageDecoderPlugin: Invalid segment retention flags"); } - for (auto const& [i, referred_to_segment_number] : enumerate(header.referred_to_segment_numbers)) { - // Quirk: t89-halftone/*-stripe.jb2 have one PatternDictionary and then one ImmediateHalftoneRegion per stripe, - // but each ImmediateHalftoneRegion (incorrectly?) sets the retention flag for the PatternDictionary to 0. - if (dead_segments.contains(referred_to_segment_number) && !context.is_power_jbig2_file) + for (auto const& [i, referred_to_segment] : enumerate(segment.referred_to_segments)) { + bool allow_reference_to_dead_segment_quirk = false; + if (context.allow_power_jbig2_quirks + && referred_to_segment->type() == JBIG2::SegmentType::PatternDictionary) { + // Quirk: t89-halftone/*-stripe.jb2 have one PatternDictionary and then one ImmediateHalftoneRegion per stripe, + // but each ImmediateHalftoneRegion (incorrectly?) sets the retention flag for the PatternDictionary to 0. + allow_reference_to_dead_segment_quirk = true; + } + if (context.options.strictness == JBIG2DecoderOptions::Strictness::Permissive + && referred_to_segment->type() == JBIG2::SegmentType::SymbolDictionary) { + // Quirk: jbig2enc (used e.g. by Google Books) sometimes generates SymbolDictionary segments that do not + // have their retention flag set, https://github.com/agl/jbig2enc/issues/121. + allow_reference_to_dead_segment_quirk = true; + } + if (dead_segments.contains(referred_to_segment->header.segment_number) && !allow_reference_to_dead_segment_quirk) return Error::from_string_literal("JBIG2ImageDecoderPlugin: Segment refers to dead segment"); auto const referred_to_segment_retention_flag = header.referred_to_segment_retention_flags[i]; if (referred_to_segment_retention_flag) { - if (dead_segments.contains(referred_to_segment_number)) + if (dead_segments.contains(referred_to_segment->header.segment_number)) return Error::from_string_literal("JBIG2ImageDecoderPlugin: Segment retention flags tried to revive dead segment"); } else { - dead_segments.set(referred_to_segment_number); + dead_segments.set(referred_to_segment->header.segment_number); } } } @@ -649,7 +675,7 @@ static ErrorOr validate_segment_header_page_associations(JBIG2LoadingConte return Error::from_string_literal("JBIG2ImageDecoderPlugin: Region, page information, end of page, or end of stripe segment with no page association"); } // Quirk: `042_*.jb2`, `amb_*.jb2` in the Power JBIG2 test suite incorrectly (cf 7.3.2) associate EndOfFile with a page. - if (segment.type() == JBIG2::SegmentType::EndOfFile && segment.header.page_association != 0 && !context.is_power_jbig2_file) + if (segment.type() == JBIG2::SegmentType::EndOfFile && segment.header.page_association != 0 && !context.allow_power_jbig2_quirks) return Error::from_string_literal("JBIG2ImageDecoderPlugin: End of file segment with page association"); // "If a segment is not associated with any page, then it must not refer to any segment that is associated with any page." @@ -835,7 +861,7 @@ static ErrorOr scan_for_page_size(JBIG2LoadingContext& context) continue; // Quirk: `042_*.jb2`, `amb_*.jb2` in the Power JBIG2 test suite incorrectly (cf 7.3.2) associate EndOfFile with a page. - if (segment.type() == JBIG2::SegmentType::EndOfFile && context.is_power_jbig2_file) + if (segment.type() == JBIG2::SegmentType::EndOfFile && context.allow_power_jbig2_quirks) continue; if (found_end_of_page) @@ -1111,16 +1137,16 @@ static ErrorOr> generic_region_decoding_procedure(Ge // independent from other contexts in the spec. They are passed in to this function. // Figure 8 – Reused context for coding the SLTP value when GBTEMPLATE is 0 - constexpr u16 sltp_context_for_template_0 = 0b10011'0110010'0101; + constexpr u16 sltp_context_for_template_0 = 0b0011'001'11001'0101; // Figure 9 – Reused context for coding the SLTP value when GBTEMPLATE is 1 - constexpr u16 sltp_context_for_template_1 = 0b0011'110010'101; + constexpr u16 sltp_context_for_template_1 = 0b0'0011'11001'101; // Figure 10 – Reused context for coding the SLTP value when GBTEMPLATE is 2 - constexpr u16 sltp_context_for_template_2 = 0b001'11001'01; + constexpr u16 sltp_context_for_template_2 = 0b1'001'1100'01; // Figure 11 – Reused context for coding the SLTP value when GBTEMPLATE is 3 - constexpr u16 sltp_context_for_template_3 = 0b011001'0101; + constexpr u16 sltp_context_for_template_3 = 0b1'01100'0101; u16 sltp_context = [](u8 gb_template) { if (gb_template == 0) @@ -1196,17 +1222,8 @@ struct GenericRefinementRegionDecodingInputParameters { Array adaptive_template_pixels {}; // "GRATX" / "GRATY" in spec. }; -struct RefinementContexts { - explicit RefinementContexts(u8 refinement_template) - { - contexts.resize(1 << (refinement_template == 0 ? 13 : 10)); - } - - Vector contexts; // "GR" (+ binary suffix) in spec. -}; - // 6.3 Generic Refinement Region Decoding Procedure -static ErrorOr> generic_refinement_region_decoding_procedure(GenericRefinementRegionDecodingInputParameters& inputs, MQArithmeticDecoder& decoder, RefinementContexts& contexts) +static ErrorOr> generic_refinement_region_decoding_procedure(GenericRefinementRegionDecodingInputParameters& inputs, MQArithmeticDecoder& decoder, JBIG2::RefinementContexts& contexts) { VERIFY(inputs.gr_template == 0 || inputs.gr_template == 1); @@ -1414,7 +1431,7 @@ struct TextContexts { }; // 6.4 Text Region Decoding Procedure -static ErrorOr> text_region_decoding_procedure(TextRegionDecodingInputParameters const& inputs, Optional& text_contexts, Optional& refinement_contexts) +static ErrorOr> text_region_decoding_procedure(TextRegionDecodingInputParameters const& inputs, Optional& text_contexts, Optional& refinement_contexts) { BigEndianInputBitStream* bit_stream = nullptr; MQArithmeticDecoder* decoder = nullptr; @@ -1459,7 +1476,7 @@ static ErrorOr> text_region_decoding_procedure(TextR if (inputs.size_of_symbol_instance_strips == 1) return 0; if (inputs.uses_huffman_encoding) - return TRY(bit_stream->read_bits(ceil(log2(inputs.size_of_symbol_instance_strips)))); + return TRY(bit_stream->read_bits(AK::ceil_log2(inputs.size_of_symbol_instance_strips))); return text_contexts->instance_t_integer_decoder.decode_non_oob(*decoder); }; @@ -1576,8 +1593,6 @@ static ErrorOr> text_region_decoding_procedure(TextR // "1) Fill a bitmap SBREG, of the size given by SBW and SBH, with the SBDEFPIXEL value." auto result = TRY(BilevelImage::create(inputs.region_width, inputs.region_height)); - if (inputs.default_pixel != 0) - return Error::from_string_literal("JBIG2ImageDecoderPlugin: Cannot handle SBDEFPIXEL not equal to 0 yet"); result->fill(inputs.default_pixel != 0); // "2) Decode the initial STRIPT value as described in 6.4.6. Negate the decoded value and assign this negated value to the variable STRIPT. @@ -1750,19 +1765,17 @@ struct SymbolContexts { }; // 6.5 Symbol Dictionary Decoding Procedure -static ErrorOr> symbol_dictionary_decoding_procedure(SymbolDictionaryDecodingInputParameters const& inputs, ReadonlyBytes data) +static ErrorOr> symbol_dictionary_decoding_procedure(SymbolDictionaryDecodingInputParameters const& inputs, Optional& generic_contexts, Optional& refinement_contexts, ReadonlyBytes data) { Optional stream; Optional bit_stream; Optional decoder; - Optional generic_contexts; Optional symbol_contexts; if (inputs.uses_huffman_encoding) { stream = FixedMemoryStream { data }; bit_stream = BigEndianInputBitStream { MaybeOwned { stream.value() } }; } else { decoder = TRY(MQArithmeticDecoder::initialize(data)); - generic_contexts = JBIG2::GenericContexts { inputs.symbol_template }; symbol_contexts = SymbolContexts {}; } @@ -1800,7 +1813,6 @@ static ErrorOr> symbol_dictionary_decoding_procedure(Sym // 6.5.8.1 Direct-coded symbol bitmap Optional text_contexts; - Optional refinement_contexts; // This belongs in 6.5.5 1) below, but also needs to be captured by read_symbol_bitmap here. Vector new_symbols; @@ -1838,7 +1850,7 @@ static ErrorOr> symbol_dictionary_decoding_procedure(Sym // 6.5.8.2.3 Setting SBSYMCODES and SBSYMCODELEN u32 number_of_symbols = inputs.input_symbols.size() + inputs.number_of_new_symbols; // "SBNUMSYMS" in spec. - u32 code_length = ceil(log2(number_of_symbols)); // "SBSYMCODELEN" in spec. + u32 code_length = AK::ceil_log2(number_of_symbols); // "SBSYMCODELEN" in spec. JBIG2::HuffmanTable const* symbol_id_table { nullptr }; if (inputs.uses_huffman_encoding) { if (!symbol_id_table_storage.has_value()) { @@ -1850,8 +1862,6 @@ static ErrorOr> symbol_dictionary_decoding_procedure(Sym if (!text_contexts.has_value()) text_contexts = TextContexts { code_length }; - if (!refinement_contexts.has_value()) - refinement_contexts = RefinementContexts(inputs.refinement_template); if (number_of_symbol_instances > 1) { // "2) If REFAGGNINST is greater than one, then decode the bitmap itself using a text region decoding procedure @@ -2022,6 +2032,7 @@ static ErrorOr> symbol_dictionary_decoding_procedure(Sym // SYMWIDTH = 0 // TOTWIDTH = 0 // HCFIRSTSYM = NSYMSDECODED" + // NOTE: The spec means "HCHEIGHT" with "HCEIGHT" presumably. i32 delta_height = TRY(read_delta_height()); height_class_height += delta_height; u32 symbol_width = 0; @@ -2035,7 +2046,9 @@ static ErrorOr> symbol_dictionary_decoding_procedure(Sym if (!opt_delta_width.has_value()) break; - VERIFY(number_of_symbols_decoded < inputs.number_of_new_symbols); + if (number_of_symbols_decoded >= inputs.number_of_new_symbols) + return Error::from_string_literal("JBIG2ImageDecoderPlugin: Invalid bitstream, too many symbols decoded in symbol dictionary"); + // " Otherwise let DW be the decoded value and set:" // SYMWIDTH = SYMWIDTH + DW // TOTWIDTH = TOTWIDTH + SYMWIDTH" @@ -2049,13 +2062,13 @@ static ErrorOr> symbol_dictionary_decoding_procedure(Sym // FIXME: Doing this eagerly is pretty wasteful. Decode on demand instead? if (!inputs.uses_huffman_encoding || inputs.uses_refinement_or_aggregate_coding) { auto bitmap = TRY(read_symbol_bitmap(symbol_width, height_class_height)); - new_symbols.append(bitmap->as_subbitmap()); + TRY(new_symbols.try_append(bitmap->as_subbitmap())); } // "iii) If SDHUFF is 1 and SDREFAGG is 0, then set: // SDNEWSYMWIDTHS[NSYMSDECODED] = SYMWIDTH" if (inputs.uses_huffman_encoding && !inputs.uses_refinement_or_aggregate_coding) - new_symbol_widths.append(symbol_width); + TRY(new_symbol_widths.try_append(symbol_width)); // "iv) Set: // NSYMSDECODED = NSYMSDECODED + 1" @@ -2082,7 +2095,7 @@ static ErrorOr> symbol_dictionary_decoding_procedure(Sym for (size_t i = height_class_first_symbol; i < number_of_symbols_decoded; ++i) { auto width = new_symbol_widths[i]; IntRect symbol_rect { static_cast(current_column), 0, static_cast(width), static_cast(height_class_height) }; - new_symbols.append(collective_bitmap->subbitmap(symbol_rect)); + TRY(new_symbols.try_append(collective_bitmap->subbitmap(symbol_rect))); current_column += width; } } @@ -2144,13 +2157,13 @@ static ErrorOr> symbol_dictionary_decoding_procedure(Sym // SDEXSYMS[J] = SDINSYMS[I] // J = J + 1" if (i < inputs.input_symbols.size()) - exported_symbols.append(inputs.input_symbols[i]); + TRY(exported_symbols.try_append(inputs.input_symbols[i])); // "b) If I >= SDNUMINSYMS then set: // SDEXSYMS[J] = SDNEWSYMS[I – SDNUMINSYMS] // J = J + 1" if (i >= inputs.input_symbols.size()) - exported_symbols.append(move(new_symbols[i - inputs.input_symbols.size()])); + TRY(exported_symbols.try_append(move(new_symbols[i - inputs.input_symbols.size()]))); } if (exported_symbols.size() != inputs.number_of_exported_symbols) @@ -2287,34 +2300,23 @@ static ErrorOr> halftone_region_decoding_procedure(H Optional skip_pattern; RefPtr skip_pattern_storage; if (inputs.enable_skip) { - skip_pattern_storage = TRY(BilevelImage::create(inputs.grayscale_width, inputs.grayscale_height)); + skip_pattern_storage = TRY(JBIG2::halftone_skip_pattern({ + inputs.region_width, + inputs.region_height, + inputs.grayscale_width, + inputs.grayscale_height, + inputs.grid_origin_x_offset, + inputs.grid_origin_y_offset, + inputs.grid_vector_x, + inputs.grid_vector_y, + inputs.pattern_width, + inputs.pattern_height, + })); skip_pattern = *skip_pattern_storage; - - // 6.6.5.1 Computing HSKIP - // "1) For each value of mg between 0 and HGH – 1, beginning from 0, perform the following steps:" - for (int m_g = 0; m_g < (int)inputs.grayscale_height; ++m_g) { - // "a) For each value of ng between 0 and HGW – 1, beginning from 0, perform the following steps:" - for (int n_g = 0; n_g < (int)inputs.grayscale_width; ++n_g) { - // "i) Set: - // x = (HGX + m_g × HRY + n_g × HRX) >> 8 - // y = (HGY + m_g × HRX – n_g × HRY) >> 8" - auto x = (inputs.grid_origin_x_offset + m_g * inputs.grid_vector_y + n_g * inputs.grid_vector_x) >> 8; - auto y = (inputs.grid_origin_y_offset + m_g * inputs.grid_vector_x - n_g * inputs.grid_vector_y) >> 8; - - // "ii) If ((x + HPW <= 0) OR (x >= HBW) OR (y + HPH <= 0) OR (y >= HBH)) then set: - // HSKIP[n_g, m_g] = 1 - // Otherwise, set: - // HSKIP[n_g, m_g] = 0" - if (x + inputs.pattern_width <= 0 || x >= (int)inputs.region_width || y + inputs.pattern_height <= 0 || y >= (int)inputs.region_height) - skip_pattern_storage->set_bit(n_g, m_g, true); - else - skip_pattern_storage->set_bit(n_g, m_g, false); - } - } } // "3) Set HBPP to ⌈log2 (HNUMPATS)⌉." - u32 bits_per_pattern = ceil(log2(inputs.patterns.size())); + u32 bits_per_pattern = AK::ceil_log2(inputs.patterns.size()); // "4) Decode an image GI of size HGW by HGH with HBPP bits per pixel using the gray-scale image decoding // procedure as described in Annex C. Set the parameters to this decoding procedure as shown in Table 23. @@ -2440,14 +2442,17 @@ static ErrorOr decode_symbol_dictionary(JBIG2LoadingContext& context, Segm // but having the custom tables available is convenient for collecting huffman tables below. Vector symbols; Vector custom_tables; + SegmentData const* last_referred_to_symbol_dictionary_segment = nullptr; for (auto const* referred_to_segment : segment.referred_to_segments) { dbgln_if(JBIG2_DEBUG, "Symbol segment refers to segment id {}", referred_to_segment->header.segment_number); - if (referred_to_segment->symbols.has_value()) + if (referred_to_segment->symbols.has_value()) { symbols.extend(referred_to_segment->symbols.value()); - else if (referred_to_segment->huffman_table.has_value()) + last_referred_to_symbol_dictionary_segment = referred_to_segment; + } else if (referred_to_segment->huffman_table.has_value()) { custom_tables.append(&referred_to_segment->huffman_table.value()); - else + } else { return Error::from_string_literal("JBIG2ImageDecoderPlugin: Symbol segment referred-to segment without symbols or huffman table"); + } } // 7.4.2.1 Symbol dictionary segment data header @@ -2475,7 +2480,7 @@ static ErrorOr decode_symbol_dictionary(JBIG2LoadingContext& context, Segm u8 refinement_template_used = (flags >> 12) & 1; // "SDREFTEMPLATE" in spec. // Quirk: 042_22.jb2 does not set SDREFAGG but it does set SDREFTEMPLATE. - if (!uses_refinement_or_aggregate_coding && refinement_template_used != 0 && !context.is_power_jbig2_file) + if (!uses_refinement_or_aggregate_coding && refinement_template_used != 0 && !context.allow_power_jbig2_quirks) return Error::from_string_literal("JBIG2ImageDecoderPlugin: Invalid refinement_template_used"); if (flags & 0b1110'0000'0000'0000) @@ -2533,13 +2538,46 @@ static ErrorOr decode_symbol_dictionary(JBIG2LoadingContext& context, Segm // "2) Decode (or retrieve the results of decoding) any referred-to symbol dictionary and tables segments." // Done further up already. - // "3) If the "bitmap coding context used" bit in the header was 1, ..." - if (bitmap_coding_context_used) - return Error::from_string_literal("JBIG2ImageDecoderPlugin: Cannot decode bitmap coding context segment yet"); - + // "3) If the "bitmap coding context used" bit in the header was 1, then, as described in E.3.8, set the arithmetic + // coding statistics for the generic region and generic refinement region decoding procedures to the values + // that they contained at the end of decoding the last-referred-to symbol dictionary segment. That symbol + // dictionary segment's symbol dictionary segment data header must have had the "bitmap coding context + // retained" bit equal to 1. The values of SDHUFF, SDREFAGG, SDTEMPLATE, SDRTEMPLATE, + // and all of the AT locations (both direct and refinement) for this symbol dictionary must match the + // corresponding values from the symbol dictionary whose context values are being used." + Optional generic_contexts; + Optional refinement_contexts; + if (bitmap_coding_context_used) { + if (!last_referred_to_symbol_dictionary_segment) + return Error::from_string_literal("JBIG2ImageDecoderPlugin: \"bitmap coding context used\" bit set, but no last-referred-to symbol dictionary segment present"); + if (!last_referred_to_symbol_dictionary_segment->retained_bitmap_coding_contexts.has_value()) + return Error::from_string_literal("JBIG2ImageDecoderPlugin: \"bitmap coding context used\" bit set, but last-referred-to symbol dictionary segment did not set \"bitmap coding context retained\""); + + auto const& last_state = last_referred_to_symbol_dictionary_segment->retained_bitmap_coding_contexts.value(); + if (last_state.used_huffman_encoding != uses_huffman_encoding) + return Error::from_string_literal("JBIG2ImageDecoderPlugin: \"bitmap coding context used\" bit set, but SDHUFF values do not match"); + if (last_state.used_refinement_or_aggregate_coding != uses_refinement_or_aggregate_coding) + return Error::from_string_literal("JBIG2ImageDecoderPlugin: \"bitmap coding context used\" bit set, but SDREFAGG values do not match"); + if (last_state.symbol_template != template_used) + return Error::from_string_literal("JBIG2ImageDecoderPlugin: \"bitmap coding context used\" bit set, but SDTEMPLATE values do not match"); + if (last_state.refinement_template != refinement_template_used) + return Error::from_string_literal("JBIG2ImageDecoderPlugin: \"bitmap coding context used\" bit set, but SDRTEMPLATE values do not match"); + if (last_state.adaptive_template_pixels != adaptive_template) + return Error::from_string_literal("JBIG2ImageDecoderPlugin: \"bitmap coding context used\" bit set, but SDATX / SDATY values do not match"); + if (last_state.refinement_adaptive_template_pixels != adaptive_refinement_template) + return Error::from_string_literal("JBIG2ImageDecoderPlugin: \"bitmap coding context used\" bit set, but SDRATX / SDRATY values do not match"); + + generic_contexts = last_state.generic_contexts; + refinement_contexts = last_state.refinement_contexts; + } // "4) If the "bitmap coding context used" bit in the header was 0, then, as described in E.3.7, // reset all the arithmetic coding statistics for the generic region and generic refinement region decoding procedures to zero." - // Nothing to do. + else { + if (!uses_huffman_encoding) + generic_contexts = JBIG2::GenericContexts { template_used }; + if (uses_refinement_or_aggregate_coding) + refinement_contexts = JBIG2::RefinementContexts { refinement_template_used }; + } // "5) Reset the arithmetic coding statistics for all the contexts of all the arithmetic integer coders to zero." // We currently do this by keeping the statistics as locals in symbol_dictionary_decoding_procedure(). @@ -2559,12 +2597,22 @@ static ErrorOr decode_symbol_dictionary(JBIG2LoadingContext& context, Segm inputs.adaptive_template_pixels = adaptive_template; inputs.refinement_template = refinement_template_used; inputs.refinement_adaptive_template_pixels = adaptive_refinement_template; - auto result = TRY(symbol_dictionary_decoding_procedure(inputs, segment.data.slice(TRY(stream.tell())))); + auto result = TRY(symbol_dictionary_decoding_procedure(inputs, generic_contexts, refinement_contexts, segment.data.slice(TRY(stream.tell())))); // "7) If the "bitmap coding context retained" bit in the header was 1, then, as described in E.3.8, preserve the current contents // of the arithmetic coding statistics for the generic region and generic refinement region decoding procedures." - if (bitmap_coding_context_retained) - return Error::from_string_literal("JBIG2ImageDecoderPlugin: Cannot retain bitmap coding context yet"); + if (bitmap_coding_context_retained) { + segment.retained_bitmap_coding_contexts = { + move(generic_contexts), + move(refinement_contexts), + uses_huffman_encoding, + uses_refinement_or_aggregate_coding, + template_used, + refinement_template_used, + adaptive_template, + adaptive_refinement_template, + }; + } segment.symbols = move(result); @@ -2649,7 +2697,7 @@ static ErrorOr decode_text_region(JBIG2LoadingContext& context, Se u16 huffman_flags = TRY(stream.read_value>()); // Quirk: 042_11.jb2 has refinement huffman table bits set but the SBREFINE bit is not set. - if (!uses_refinement_coding && (huffman_flags & 0x7fc0) != 0 && !context.is_power_jbig2_file) + if (!uses_refinement_coding && (huffman_flags & 0x7fc0) != 0 && !context.allow_power_jbig2_quirks) return Error::from_string_literal("JBIG2ImageDecoderPlugin: Huffman flags have refinement bits set but refinement bit is not set"); huffman_tables = TRY(text_region_huffman_tables_from_flags(huffman_flags, custom_tables)); @@ -2758,13 +2806,13 @@ static ErrorOr decode_text_region(JBIG2LoadingContext& context, Se // Done further up, since it's needed to decode the symbol ID Huffman table already. // "3) As described in E.3.7, reset all the arithmetic coding statistics to zero." - u32 id_symbol_code_length = ceil(log2(symbols.size())); + u32 id_symbol_code_length = AK::ceil_log2(symbols.size()); Optional text_contexts; if (!uses_huffman_encoding) text_contexts = TextContexts { id_symbol_code_length }; - Optional refinement_contexts; + Optional refinement_contexts; if (uses_refinement_coding) - refinement_contexts = RefinementContexts { refinement_template }; + refinement_contexts = JBIG2::RefinementContexts { refinement_template }; // "4) Invoke the text region decoding procedure described in 6.4, with the parameters to the text region decoding procedure set as shown in Table 34." TextRegionDecodingInputParameters inputs; @@ -3149,7 +3197,7 @@ static ErrorOr decode_generic_refinement_region(JBIG2LoadingContex if (flags & 0b1111'1100) return Error::from_string_literal("JBIG2ImageDecoderPlugin: Invalid refinement flags"); - dbgln_if(JBIG2_DEBUG, "GRTEMPLATE={} TPRDON={}", arithmetic_coding_template, typical_prediction_generic_refinement_on); + dbgln_if(JBIG2_DEBUG, "GRTEMPLATE={} TPGRON={}", arithmetic_coding_template, typical_prediction_generic_refinement_on); // 7.4.7.3 Generic refinement region segment AT flags Array adaptive_template_pixels {}; @@ -3186,20 +3234,23 @@ static ErrorOr decode_generic_refinement_region(JBIG2LoadingContex } // "2) As described in E.3.7, reset all the arithmetic coding statistics to zero." - RefinementContexts contexts { arithmetic_coding_template }; + JBIG2::RefinementContexts contexts { arithmetic_coding_template }; // "3) Determine the buffer associated with the region segment that this segment refers to." // Details described in 7.4.7.4 Reference bitmap selection. - BilevelImage const* reference_bitmap = nullptr; - if (segment.referred_to_segments.size() == 1) { - reference_bitmap = segment.referred_to_segments[0]->aux_buffer.ptr(); - VERIFY(reference_bitmap->width() == segment.referred_to_segments[0]->aux_buffer_information_field.width); - VERIFY(reference_bitmap->height() == segment.referred_to_segments[0]->aux_buffer_information_field.height); - } else { - // When adding support for this and for intermediate generic refinement regions, make sure to only allow - // this case for immediate generic refinement regions. - return Error::from_string_literal("JBIG2ImageDecoderPlugin: Generic refinement region without reference segment not yet implemented"); - } + BilevelSubImage reference_bitmap = [&]() { + if (segment.referred_to_segments.size() == 1) { + auto reference_bitmap = segment.referred_to_segments[0]->aux_buffer; + VERIFY(reference_bitmap->width() == segment.referred_to_segments[0]->aux_buffer_information_field.width); + VERIFY(reference_bitmap->height() == segment.referred_to_segments[0]->aux_buffer_information_field.height); + return reference_bitmap->as_subbitmap(); + } + + // Enforced by validate_segment_header_references() earlier. + VERIFY(segment.type() != JBIG2::SegmentType::IntermediateGenericRefinementRegion); + + return context.page.bits->subbitmap(information_field.rect()); + }(); // "4) Invoke the generic refinement region decoding procedure described in 6.3, with the parameters to the // generic refinement region decoding procedure set as shown in Table 38." @@ -3208,8 +3259,7 @@ static ErrorOr decode_generic_refinement_region(JBIG2LoadingContex inputs.region_width = information_field.width; inputs.region_height = information_field.height; inputs.gr_template = arithmetic_coding_template; - auto subbitmap = reference_bitmap->as_subbitmap(); - inputs.reference_bitmap = &subbitmap; + inputs.reference_bitmap = &reference_bitmap; inputs.reference_x_offset = 0; inputs.reference_y_offset = 0; inputs.is_typical_prediction_used = typical_prediction_generic_refinement_on; @@ -3666,7 +3716,7 @@ ErrorOr JBIG2ImageDecoderPlugin::frame(size_t index, Optio return ImageFrameDescriptor { move(bitmap), 0 }; } -ErrorOr JBIG2ImageDecoderPlugin::decode_embedded(Vector data) +ErrorOr> JBIG2ImageDecoderPlugin::create_embedded_jbig2_decoder(Vector data) { auto plugin = TRY(adopt_nonnull_own_or_enomem(new (nothrow) JBIG2ImageDecoderPlugin({}))); plugin->m_context->organization = JBIG2::Organization::Embedded; @@ -3682,8 +3732,29 @@ ErrorOr JBIG2ImageDecoderPlugin::decode_embedded(Vectorm_context)); + return plugin; +} + +ErrorOr> JBIG2ImageDecoderPlugin::decode_embedded(Vector data) +{ + auto plugin = TRY(create_embedded_jbig2_decoder(data)); + return *plugin->m_context->page.bits; +} + +ErrorOr> JBIG2ImageDecoderPlugin::decode_embedded_intermediate_region_segment(Vector data, u32 segment_number) +{ + auto plugin = TRY(create_embedded_jbig2_decoder(data)); + + auto target_segment = plugin->m_context->segments.find_if([&segment_number](auto const& segment) { + return segment.header.segment_number == segment_number; + }); + if (target_segment == plugin->m_context->segments.end()) + return Error::from_string_literal("JBIG2ImageDecoderPlugin: Segment number not found in embedded JBIG2 data"); + + if (!is_intermediate_region_segment(target_segment->type())) + return Error::from_string_literal("JBIG2ImageDecoderPlugin: Target segment is not an intermediate region segment"); - return plugin->m_context->page.bits->to_byte_buffer(); + return *target_segment->aux_buffer; } } diff --git a/Userland/Libraries/LibGfx/ImageFormats/JBIG2Loader.h b/Userland/Libraries/LibGfx/ImageFormats/JBIG2Loader.h index 20395377a7512e..84eb4d13f2ce3b 100644 --- a/Userland/Libraries/LibGfx/ImageFormats/JBIG2Loader.h +++ b/Userland/Libraries/LibGfx/ImageFormats/JBIG2Loader.h @@ -12,6 +12,7 @@ namespace Gfx { +class BilevelImage; struct JBIG2LoadingContext; struct MQArithmeticCoderContext; class MQArithmeticDecoder; @@ -22,6 +23,15 @@ struct JBIG2DecoderOptions { Yes, }; LogComments log_comments { LogComments::Yes }; + + enum class Strictness { + // Accepts images that violate the JBIG2 spec in ways found in the wild. + Permissive, + + // Rejects images that do not conform to the JBIG2 spec. + SpecCompliant, + }; + Strictness strictness { Strictness::Permissive }; }; namespace JBIG2 { @@ -69,11 +79,14 @@ class JBIG2ImageDecoderPlugin : public ImageDecoderPlugin { virtual size_t frame_count() override; virtual ErrorOr frame(size_t index, Optional ideal_size = {}) override; - static ErrorOr decode_embedded(Vector); + static ErrorOr> decode_embedded(Vector); + static ErrorOr> decode_embedded_intermediate_region_segment(Vector, u32 segment_number); private: JBIG2ImageDecoderPlugin(JBIG2DecoderOptions); + static ErrorOr> create_embedded_jbig2_decoder(Vector data); + OwnPtr m_context; }; diff --git a/Userland/Libraries/LibGfx/ImageFormats/JBIG2Shared.cpp b/Userland/Libraries/LibGfx/ImageFormats/JBIG2Shared.cpp index d826fa4e30fecd..b4158a84e0c805 100644 --- a/Userland/Libraries/LibGfx/ImageFormats/JBIG2Shared.cpp +++ b/Userland/Libraries/LibGfx/ImageFormats/JBIG2Shared.cpp @@ -10,6 +10,35 @@ namespace Gfx::JBIG2 { +ErrorOr> halftone_skip_pattern(HalftoneGeometry const& inputs) +{ + auto skip_pattern = TRY(BilevelImage::create(inputs.grayscale_width, inputs.grayscale_height)); + + // 6.6.5.1 Computing HSKIP + // "1) For each value of mg between 0 and HGH – 1, beginning from 0, perform the following steps:" + for (int m_g = 0; m_g < (int)inputs.grayscale_height; ++m_g) { + // "a) For each value of ng between 0 and HGW – 1, beginning from 0, perform the following steps:" + for (int n_g = 0; n_g < (int)inputs.grayscale_width; ++n_g) { + // "i) Set: + // x = (HGX + m_g × HRY + n_g × HRX) >> 8 + // y = (HGY + m_g × HRX – n_g × HRY) >> 8" + auto x = (inputs.grid_origin_x_offset + m_g * inputs.grid_vector_y + n_g * inputs.grid_vector_x) >> 8; + auto y = (inputs.grid_origin_y_offset + m_g * inputs.grid_vector_x - n_g * inputs.grid_vector_y) >> 8; + + // "ii) If ((x + HPW <= 0) OR (x >= HBW) OR (y + HPH <= 0) OR (y >= HBH)) then set: + // HSKIP[n_g, m_g] = 1 + // Otherwise, set: + // HSKIP[n_g, m_g] = 0" + if (x + inputs.pattern_width <= 0 || x >= (int)inputs.region_width || y + inputs.pattern_height <= 0 || y >= (int)inputs.region_height) + skip_pattern->set_bit(n_g, m_g, true); + else + skip_pattern->set_bit(n_g, m_g, false); + } + } + + return skip_pattern; +} + ErrorOr text_region_huffman_tables_from_flags(u16 huffman_flags, Vector custom_tables) { TextRegionHuffmanTables tables; @@ -158,7 +187,7 @@ ErrorOr symbol_dictionary_huffman_tables_from_fla else if (huffman_table_selection_for_width_differences == 1) tables.delta_width_table = TRY(JBIG2::HuffmanTable::standard_huffman_table(JBIG2::HuffmanTable::StandardTable::B_3)); else if (huffman_table_selection_for_width_differences == 2) - return Error::from_string_literal("JBIG2: Invalid huffman_table_selection_for_height_differences"); + return Error::from_string_literal("JBIG2: Invalid huffman_table_selection_for_width_differences"); else if (huffman_table_selection_for_width_differences == 3) tables.delta_width_table = TRY(custom_table()); } diff --git a/Userland/Libraries/LibGfx/ImageFormats/JBIG2Shared.h b/Userland/Libraries/LibGfx/ImageFormats/JBIG2Shared.h index 76faa9cc682208..ddf4514f53f260 100644 --- a/Userland/Libraries/LibGfx/ImageFormats/JBIG2Shared.h +++ b/Userland/Libraries/LibGfx/ImageFormats/JBIG2Shared.h @@ -10,10 +10,26 @@ #include #include #include +#include #include +#include namespace Gfx::JBIG2 { +struct HalftoneGeometry { + u32 region_width { 0 }; // "HBW" in spec. + u32 region_height { 0 }; // "HBH" in spec. + u32 grayscale_width { 0 }; // "HGW" in spec. + u32 grayscale_height { 0 }; // "HGH" in spec. + i32 grid_origin_x_offset { 0 }; // "HGX" in spec. + i32 grid_origin_y_offset { 0 }; // "HGY" in spec. + u16 grid_vector_x { 0 }; // "HRY" in spec. + u16 grid_vector_y { 0 }; // "HRX" in spec. + u8 pattern_width { 0 }; // "HPW" in spec. + u8 pattern_height { 0 }; // "HPH" in spec. +}; +ErrorOr> halftone_skip_pattern(HalftoneGeometry const&); + class HuffmanTable; struct TextRegionHuffmanTables { @@ -118,6 +134,11 @@ struct [[gnu::packed]] RegionSegmentInformationField { BigEndian y_location; u8 flags { 0 }; + IntRect rect() const + { + return { x_location, y_location, width, height }; + } + CombinationOperator external_combination_operator() const { VERIFY((flags & 0x7) <= 4); @@ -134,6 +155,8 @@ static_assert(AssertSize()); struct AdaptiveTemplatePixel { i8 x { 0 }; i8 y { 0 }; + + bool operator<=>(AdaptiveTemplatePixel const&) const = default; }; // Figure 7 – Field to which AT pixel locations are restricted @@ -167,6 +190,15 @@ struct GenericContexts { } }; +struct RefinementContexts { + explicit RefinementContexts(u8 refinement_template) + { + contexts.resize(1 << (refinement_template == 0 ? 13 : 10)); + } + + Vector contexts; // "GR" (+ binary suffix) in spec. +}; + // 7.4.8 Page information segment syntax struct [[gnu::packed]] PageInformationSegment { BigEndian bitmap_width; diff --git a/Userland/Libraries/LibGfx/ImageFormats/JBIG2Writer.cpp b/Userland/Libraries/LibGfx/ImageFormats/JBIG2Writer.cpp index bce1e7a6bb9b08..b72695f2cdd866 100644 --- a/Userland/Libraries/LibGfx/ImageFormats/JBIG2Writer.cpp +++ b/Userland/Libraries/LibGfx/ImageFormats/JBIG2Writer.cpp @@ -21,6 +21,7 @@ #include #include #include +#include #include #include #include @@ -171,9 +172,6 @@ static ErrorOr generic_region_encoding_procedure(GenericRegionEncodingInpu if (inputs.skip_pattern.has_value() && (inputs.skip_pattern->width() != width || inputs.skip_pattern->height() != height)) return Error::from_string_literal("JBIG2Writer: Invalid USESKIP dimensions"); - if (inputs.skip_pattern.has_value()) - return Error::from_string_literal("JBIG2Writer: Cannot encode USESKIP yet"); - static constexpr auto get_pixel = [](BilevelImage const& buffer, int x, int y) -> bool { // 6.2.5.2 Coding order and edge conventions // "• All pixels lying outside the bounds of the actual bitmap have the value 0." @@ -256,16 +254,16 @@ static ErrorOr generic_region_encoding_procedure(GenericRegionEncodingInpu // independent from other contexts in the spec. They are passed in to this function. // Figure 8 – Reused context for coding the SLTP value when GBTEMPLATE is 0 - constexpr u16 sltp_context_for_template_0 = 0b10011'0110010'0101; + constexpr u16 sltp_context_for_template_0 = 0b0011'001'11001'0101; // Figure 9 – Reused context for coding the SLTP value when GBTEMPLATE is 1 - constexpr u16 sltp_context_for_template_1 = 0b0011'110010'101; + constexpr u16 sltp_context_for_template_1 = 0b0'0011'11001'101; // Figure 10 – Reused context for coding the SLTP value when GBTEMPLATE is 2 - constexpr u16 sltp_context_for_template_2 = 0b001'11001'01; + constexpr u16 sltp_context_for_template_2 = 0b1'001'1100'01; // Figure 11 – Reused context for coding the SLTP value when GBTEMPLATE is 3 - constexpr u16 sltp_context_for_template_3 = 0b011001'0101; + constexpr u16 sltp_context_for_template_3 = 0b1'01100'0101; u16 sltp_context = [](u8 gb_template) { if (gb_template == 0) @@ -288,13 +286,16 @@ static ErrorOr generic_region_encoding_procedure(GenericRegionEncodingInpu // " 2) Create a bitmap GBREG of width GBW and height GBH pixels." // auto result = TRY(BilevelImage::create(inputs.region_width, inputs.region_height)); + // skip_pattern can only set when this is called for encoding halftone regions. + // Halftone regions never set is_typical_prediction_used. + VERIFY(!inputs.is_typical_prediction_used || !inputs.skip_pattern.has_value()); + // "3) Decode each row as follows:" for (size_t y = 0; y < height; ++y) { // "a) If all GBH rows have been decoded then the decoding is complete; proceed to step 4)." // "b) If TPGDON is 1, then decode a bit using the arithmetic entropy coder..." if (inputs.is_typical_prediction_used) { // "i) If the current row of GBREG is identical to the row immediately above, then SLTP = 1; otherwise SLTP = 0." - // FIXME: If skip_pattern is set, we should probably ignore skipped pixels here. bool is_line_identical_to_previous_line = true; for (size_t x = 0; x < width; ++x) { if (inputs.image.get_bit(x, y) != get_pixel(inputs.image, (int)x, (int)y - 1)) { @@ -337,26 +338,17 @@ namespace { struct GenericRefinementRegionEncodingInputParameters { BilevelImage const& image; // Of dimensions "GRW" x "GRH" in spec terms. u8 gr_template { 0 }; // "GRTEMPLATE" in spec. - BilevelImage const* reference_bitmap { nullptr }; // "GRREFERENCE" in spec. + BilevelSubImage reference_bitmap; // "GRREFERENCE" in spec. i32 reference_x_offset { 0 }; // "GRREFERENCEDX" in spec. i32 reference_y_offset { 0 }; // "GRREFERENCEDY" in spec. bool is_typical_prediction_used { false }; // "TPGRON" in spec. Array adaptive_template_pixels {}; // "GRATX" / "GRATY" in spec. }; -struct RefinementContexts { - explicit RefinementContexts(u8 refinement_template) - { - contexts.resize(1 << (refinement_template == 0 ? 13 : 10)); - } - - Vector contexts; // "GR" (+ binary suffix) in spec. -}; - } // 6.3 Generic Refinement Region Decoding Procedure, but in reverse. -static ErrorOr generic_refinement_region_encoding_procedure(GenericRefinementRegionEncodingInputParameters& inputs, MQArithmeticEncoder& encoder, RefinementContexts& contexts) +static ErrorOr generic_refinement_region_encoding_procedure(GenericRefinementRegionEncodingInputParameters& inputs, MQArithmeticEncoder& encoder, JBIG2::RefinementContexts& contexts) { // FIXME: Try to come up with a way to share more code with generic_refinement_region_decoding_procedure(). auto width = inputs.image.width(); @@ -378,7 +370,7 @@ static ErrorOr generic_refinement_region_encoding_procedure(GenericRefinem }; // Figure 12 – 13-pixel refinement template showing the AT pixels at their nominal locations - constexpr auto compute_context_0 = [](ReadonlySpan adaptive_pixels, BilevelImage const& reference, int reference_x, int reference_y, BilevelImage const& buffer, int x, int y) -> u16 { + constexpr auto compute_context_0 = [](ReadonlySpan adaptive_pixels, BilevelSubImage const& reference, int reference_x, int reference_y, BilevelImage const& buffer, int x, int y) -> u16 { u16 result = 0; for (int dy = -1; dy <= 1; ++dy) { @@ -399,7 +391,7 @@ static ErrorOr generic_refinement_region_encoding_procedure(GenericRefinem }; // Figure 13 – 10-pixel refinement template - constexpr auto compute_context_1 = [](ReadonlySpan, BilevelImage const& reference, int reference_x, int reference_y, BilevelImage const& buffer, int x, int y) -> u16 { + constexpr auto compute_context_1 = [](ReadonlySpan, BilevelSubImage const& reference, int reference_x, int reference_y, BilevelImage const& buffer, int x, int y) -> u16 { u16 result = 0; for (int dy = -1; dy <= 1; ++dy) { @@ -440,10 +432,10 @@ static ErrorOr generic_refinement_region_encoding_procedure(GenericRefinem auto predict = [&](size_t x, size_t y) -> Optional { // "• a 3 × 3 pixel array in the reference bitmap (Figure 16), centred at the location // corresponding to the current pixel, contains pixels all of the same value." - bool prediction = get_pixel(*inputs.reference_bitmap, x - inputs.reference_x_offset - 1, y - inputs.reference_y_offset - 1); + bool prediction = get_pixel(inputs.reference_bitmap, x - inputs.reference_x_offset - 1, y - inputs.reference_y_offset - 1); for (int dy = -1; dy <= 1; ++dy) for (int dx = -1; dx <= 1; ++dx) - if (get_pixel(*inputs.reference_bitmap, x - inputs.reference_x_offset + dx, y - inputs.reference_y_offset + dy) != prediction) + if (get_pixel(inputs.reference_bitmap, x - inputs.reference_x_offset + dx, y - inputs.reference_y_offset + dy) != prediction) return {}; return prediction; }; @@ -471,7 +463,7 @@ static ErrorOr generic_refinement_region_encoding_procedure(GenericRefinem // "c) If LTP = 0 then, from left to right, explicitly decode all pixels of the current row of GRREG. The // procedure for each pixel is as follows:" for (size_t x = 0; x < width; ++x) { - u16 context = compute_context(inputs.adaptive_template_pixels, *inputs.reference_bitmap, x - inputs.reference_x_offset, y - inputs.reference_y_offset, inputs.image, x, y); + u16 context = compute_context(inputs.adaptive_template_pixels, inputs.reference_bitmap, x - inputs.reference_x_offset, y - inputs.reference_y_offset, inputs.image, x, y); encoder.encode_bit(inputs.image.get_bit(x, y), contexts.contexts[context]); } } else { @@ -484,7 +476,7 @@ static ErrorOr generic_refinement_region_encoding_procedure(GenericRefinem // TPGRON must be 1 if LTP is set. (The spec has an explicit "TPGRON is 1 AND" check here, but it is pointless.) VERIFY(inputs.is_typical_prediction_used); if (!prediction.has_value()) { - u16 context = compute_context(inputs.adaptive_template_pixels, *inputs.reference_bitmap, x - inputs.reference_x_offset, y - inputs.reference_y_offset, inputs.image, x, y); + u16 context = compute_context(inputs.adaptive_template_pixels, inputs.reference_bitmap, x - inputs.reference_x_offset, y - inputs.reference_y_offset, inputs.image, x, y); encoder.encode_bit(inputs.image.get_bit(x, y), contexts.contexts[context]); } } @@ -494,16 +486,26 @@ static ErrorOr generic_refinement_region_encoding_procedure(GenericRefinem return {}; } -static ErrorOr> symbol_image(JBIG2::SymbolDictionarySegmentData::HeightClass::Symbol const& symbol) +static ErrorOr> symbol_image(JBIG2::SymbolDictionarySegmentData::HeightClass::Symbol const& symbol, Vector const& symbols) { if (symbol.image.has>()) return symbol.image.get>(); if (symbol.image.has()) return symbol.image.get().refines_to; + auto image = TRY(BilevelImage::create(symbol.size.width(), symbol.size.height())); + image->fill(false); auto const& text_strips = symbol.image.get(); - (void)text_strips; - return Error::from_string_literal("JBIG2Writer: Cannot write refinements of refinements by text strips yet"); + for (auto const& strip : text_strips.strips) { + for (auto const& instance : strip.symbol_instances) { + if (instance.symbol_id >= symbols.size()) + return Error::from_string_literal("JBIG2Writer: Invalid symbol ID in text strip symbol instance"); + auto const& instance_symbol = symbols[instance.symbol_id]; + auto instance_image = TRY(symbol_image(instance_symbol, symbols)); + instance_image->composite_onto(image, { instance.s, instance.t }, BilevelImage::CompositionType::Or); + } + } + return image; } namespace { @@ -574,7 +576,7 @@ struct TextContexts { } // 6.4 Text Region Decoding Procedure, but in reverse. -static ErrorOr text_region_encoding_procedure(TextRegionEncodingInputParameters const& inputs, Optional& text_contexts, Optional& refinement_contexts) +static ErrorOr text_region_encoding_procedure(TextRegionEncodingInputParameters const& inputs, Optional& text_contexts, Optional& refinement_contexts) { BigEndianOutputBitStream* bit_stream = nullptr; MQArithmeticEncoder* encoder = nullptr; @@ -624,13 +626,12 @@ static ErrorOr text_region_encoding_procedure(TextRegionEncodingInputParam // • If SBHUFF is 1, decode a value by reading ceil(log2(SBSTRIPS)) bits directly from the bitstream. // • If SBHUFF is 0, decode a value using the IAIT integer arithmetic decoding procedure (see Annex A)." auto write_instance_t = [&](i32 value) -> ErrorOr { - // FIXME: The spec wants this check for all valid strip sizes (1, 2, 4, 8). - if (inputs.size_of_symbol_instance_strips == 1 && value >= static_cast(inputs.size_of_symbol_instance_strips)) + if (value < 0 || value >= static_cast(inputs.size_of_symbol_instance_strips)) return Error::from_string_literal("JBIG2Writer: Symbol instance T coordinate out of range"); if (inputs.size_of_symbol_instance_strips == 1) return {}; if (inputs.uses_huffman_encoding) - return TRY(bit_stream->write_bits((u64)value, ceil(log2(inputs.size_of_symbol_instance_strips)))); + return TRY(bit_stream->write_bits((u64)value, AK::ceil_log2(inputs.size_of_symbol_instance_strips))); return text_contexts->instance_t_integer_encoder.encode_non_oob(*encoder, value); }; @@ -723,10 +724,10 @@ static ErrorOr text_region_encoding_procedure(TextRegionEncodingInputParam if (static_cast(symbol.size.height()) + symbol_instance.refinement_data->delta_height < 0) return Error::from_string_literal("JBIG2Writer: Refinement height out of bounds"); - auto reference_bitmap = TRY(symbol_image(symbol)); + auto reference_bitmap = TRY(symbol_image(symbol, inputs.symbols)); GenericRefinementRegionEncodingInputParameters refinement_inputs { .image = symbol_instance.refinement_data->refines_to, - .reference_bitmap = reference_bitmap, + .reference_bitmap = reference_bitmap->as_subbitmap(), }; // FIXME: Instead, just compute the delta here instead of having it be passed in? @@ -940,19 +941,17 @@ struct SymbolContexts { } // 6.5 Symbol Dictionary Decoding Procedure, but in reverse. -static ErrorOr symbol_dictionary_encoding_procedure(SymbolDictionaryEncodingInputParameters const& inputs, Vector& exported_symbols) +static ErrorOr symbol_dictionary_encoding_procedure(SymbolDictionaryEncodingInputParameters const& inputs, Optional& generic_contexts, Optional& refinement_contexts, Vector& exported_symbols) { Optional stream; Optional bit_stream; Optional encoder; - Optional generic_contexts; Optional symbol_contexts; if (inputs.uses_huffman_encoding) { stream = AllocatingMemoryStream {}; bit_stream = BigEndianOutputBitStream { MaybeOwned { stream.value() } }; } else { encoder = TRY(MQArithmeticEncoder::initialize(0)); - generic_contexts = JBIG2::GenericContexts { inputs.symbol_template }; symbol_contexts = SymbolContexts {}; } @@ -991,7 +990,6 @@ static ErrorOr symbol_dictionary_encoding_procedure(SymbolDictionary // 6.5.8.1 Direct-coded symbol bitmap Optional text_contexts; - Optional refinement_contexts; // This belongs in 6.5.5 1) below, but also needs to be captured by write_symbol_bitmap here. Vector new_symbols; @@ -1042,7 +1040,7 @@ static ErrorOr symbol_dictionary_encoding_procedure(SymbolDictionary // 6.5.8.2.3 Setting SBSYMCODES and SBSYMCODELEN u32 number_of_symbols = inputs.input_symbols.size() + inputs.number_of_new_symbols; // "SBNUMSYMS" in spec. - u32 code_length = ceil(log2(number_of_symbols)); // "SBSYMCODELEN" in spec. + u32 code_length = AK::ceil_log2(number_of_symbols); // "SBSYMCODELEN" in spec. JBIG2::HuffmanTable const* symbol_id_table { nullptr }; if (inputs.uses_huffman_encoding) { if (!symbol_id_table_storage.has_value()) { @@ -1054,8 +1052,6 @@ static ErrorOr symbol_dictionary_encoding_procedure(SymbolDictionary if (!text_contexts.has_value()) text_contexts = TextContexts { code_length }; - if (!refinement_contexts.has_value()) - refinement_contexts = RefinementContexts(inputs.refinement_template); if (number_of_symbol_instances > 1) { auto const& refines_using_strips = symbol.image.get(); @@ -1122,7 +1118,10 @@ static ErrorOr symbol_dictionary_encoding_procedure(SymbolDictionary if (refinement_image.symbol_id >= inputs.input_symbols.size() && refinement_image.symbol_id - inputs.input_symbols.size() >= new_symbols.size()) return Error::from_string_literal("JBIG2Writer: Refinement/aggregate symbol ID out of range"); - auto const& IBO = TRY(symbol_image(refinement_image.symbol_id < inputs.input_symbols.size() ? inputs.input_symbols[refinement_image.symbol_id] : new_symbols[refinement_image.symbol_id - inputs.input_symbols.size()])); + Vector all_symbols; + all_symbols.extend(inputs.input_symbols); + all_symbols.extend(new_symbols); + auto const& IBO = TRY(symbol_image(all_symbols[refinement_image.symbol_id], all_symbols)); MQArithmeticEncoder* refinement_encoder = nullptr; Optional huffman_refinement_encoder; @@ -1136,7 +1135,7 @@ static ErrorOr symbol_dictionary_encoding_procedure(SymbolDictionary // Table 18 – Parameters used to decode a symbol's bitmap when REFAGGNINST = 1 GenericRefinementRegionEncodingInputParameters refinement_inputs { .image = *refinement_image.refines_to, - .reference_bitmap = IBO, + .reference_bitmap = IBO->as_subbitmap(), }; refinement_inputs.gr_template = inputs.refinement_template; refinement_inputs.reference_x_offset = refinement_image.delta_x_offset; @@ -1537,13 +1536,33 @@ static ErrorOr encode_jbig2_header(Stream& stream, JBIG2::FileHeaderData c namespace { +struct SerializedSegmentData { + ByteBuffer data; + size_t header_size { 0 }; +}; + struct JBIG2EncodingContext { + JBIG2EncodingContext(Vector const& segments) + : segments(segments) + { + } + + Vector const& segments; + HashMap segment_by_id; + HashMap segment_data_by_id; + HashMap> codes_by_segment_id; HashMap tables_by_segment_id; HashMap> symbols_by_segment_id; + + struct BitmapCodingContextState { + Optional generic_contexts; + Optional refinement_contexts; + }; + HashMap retained_bitmap_coding_contexts; }; } @@ -1689,6 +1708,7 @@ static ErrorOr encode_symbol_dictionary(JBIG2::SymbolDictionarySegmentData // Get referred-to symbol and table segments off header.referred_to_segments. Vector custom_tables; Vector input_symbols; + Optional last_referred_to_symbol_dictionary_segment_id; for (auto const& referred_to_segment_number : header.referred_to_segments) { auto maybe_segment = context.segment_by_id.get(referred_to_segment_number.segment_number); if (!maybe_segment.has_value()) @@ -1706,6 +1726,7 @@ static ErrorOr encode_symbol_dictionary(JBIG2::SymbolDictionarySegmentData if (!maybe_symbols.has_value()) return Error::from_string_literal("JBIG2Writer: Could not find referred-to symbols for text region"); input_symbols.extend(maybe_symbols.value()); + last_referred_to_symbol_dictionary_segment_id = referred_to_segment_number.segment_number; continue; } } @@ -1713,6 +1734,8 @@ static ErrorOr encode_symbol_dictionary(JBIG2::SymbolDictionarySegmentData // 7.4.2 Symbol dictionary segment syntax bool uses_huffman_encoding = (symbol_dictionary.flags & 1) != 0; bool uses_refinement_or_aggregate_coding = (symbol_dictionary.flags & 2) != 0; + bool bitmap_coding_context_used = (symbol_dictionary.flags >> 8) & 1; + bool bitmap_coding_context_retained = (symbol_dictionary.flags >> 9) & 1; u8 symbol_template = (symbol_dictionary.flags >> 10) & 3; u8 symbol_refinement_template = (symbol_dictionary.flags >> 12) & 1; @@ -1744,8 +1767,28 @@ static ErrorOr encode_symbol_dictionary(JBIG2::SymbolDictionarySegmentData inputs.refinement_adaptive_template_pixels = symbol_dictionary.refinement_adaptive_template_pixels; inputs.trailing_7fff_handling = symbol_dictionary.trailing_7fff_handling; + Optional generic_contexts; + Optional refinement_contexts; + if (bitmap_coding_context_used) { + if (!last_referred_to_symbol_dictionary_segment_id.has_value()) + return Error::from_string_literal("JBIG2Writer: \"bitmap coding context used\" bit set, but no last-referred-to symbol dictionary segment present"); + auto maybe_state = context.retained_bitmap_coding_contexts.get(last_referred_to_symbol_dictionary_segment_id.value()); + if (!maybe_state.has_value()) + return Error::from_string_literal("JBIG2Writer: \"bitmap coding context used\" bit set, but last-referred-to symbol dictionary segment did not set \"bitmap coding context retained\""); + + // Consistency of uses_huffman_encoding, uses_refinement_or_aggregate_coding, symbol_template, refinement_template and adaptive template pixels is checked by the loader. + auto const& state = maybe_state.value(); + generic_contexts = state.generic_contexts; + refinement_contexts = state.refinement_contexts; + } else { + if (!uses_huffman_encoding) + generic_contexts = JBIG2::GenericContexts { inputs.symbol_template }; + if (inputs.uses_refinement_or_aggregate_coding) + refinement_contexts = JBIG2::RefinementContexts(inputs.refinement_template); + } + Vector exported_symbols; - ByteBuffer data = TRY(symbol_dictionary_encoding_procedure(inputs, exported_symbols)); + ByteBuffer data = TRY(symbol_dictionary_encoding_procedure(inputs, generic_contexts, refinement_contexts, exported_symbols)); u32 number_of_exported_symbols = exported_symbols.size(); @@ -1767,6 +1810,14 @@ static ErrorOr encode_symbol_dictionary(JBIG2::SymbolDictionarySegmentData if (context.symbols_by_segment_id.set(header.segment_number, move(exported_symbols)) != HashSetResult::InsertedNewEntry) return Error::from_string_literal("JBIG2Writer: Duplicate symbol segment ID"); + if (bitmap_coding_context_retained) { + JBIG2EncodingContext::BitmapCodingContextState state { + move(generic_contexts), + move(refinement_contexts), + }; + VERIFY(context.retained_bitmap_coding_contexts.set(header.segment_number, move(state)) == HashSetResult::InsertedNewEntry); + } + return {}; } @@ -1930,7 +1981,7 @@ static ErrorOr encode_text_region(JBIG2::TextRegionSegmentData const& text i8 delta_s_offset = AK::sign_extend(delta_s_offset_value, 5); u8 refinement_template = (text_region.flags >> 15) != 0; - u32 id_symbol_code_length = ceil(log2(symbols.size())); + u32 id_symbol_code_length = AK::ceil_log2(symbols.size()); ByteBuffer symbol_id_huffman_decoding_table; Vector symbol_id_codes; @@ -2007,9 +2058,9 @@ static ErrorOr encode_text_region(JBIG2::TextRegionSegmentData const& text Optional text_contexts; if (!uses_huffman_encoding) text_contexts = TextContexts { id_symbol_code_length }; - Optional refinement_contexts; + Optional refinement_contexts; if (uses_refinement_coding) - refinement_contexts = RefinementContexts { refinement_template }; + refinement_contexts = JBIG2::RefinementContexts { refinement_template }; AllocatingMemoryStream output_stream; Optional encoder; @@ -2117,31 +2168,40 @@ static ErrorOr encode_halftone_region(JBIG2::HalftoneRegionSegmentData con auto const& pattern_dictionary = referred_to_segment.data.get(); // FIXME: Add a halftone_region_encoding_procedure()? For now, it's just inlined here. - u32 bits_per_pattern = ceil(log2(pattern_dictionary.gray_max + 1)); + u32 bits_per_pattern = AK::ceil_log2(pattern_dictionary.gray_max + 1); - // FIXME: Implement support for enable_skip. + // "2) If HENABLESKIP equals 1, compute a bitmap HSKIP as shown in 6.6.5.1." Optional skip_pattern; + RefPtr skip_pattern_storage; bool const enable_skip = ((halftone_region.flags >> 3) & 1) != 0; - if (enable_skip) - return Error::from_string_literal("JBIG2Writer: Halftone region skip pattern not yet implemented"); + if (enable_skip) { + skip_pattern_storage = TRY(JBIG2::halftone_skip_pattern({ + halftone_region.region_segment_information.width, + halftone_region.region_segment_information.height, + halftone_region.grayscale_width, + halftone_region.grayscale_height, + halftone_region.grid_offset_x_times_256, + halftone_region.grid_offset_y_times_256, + halftone_region.grid_vector_x_times_256, + halftone_region.grid_vector_y_times_256, + pattern_dictionary.pattern_width, + pattern_dictionary.pattern_height, + })); + skip_pattern = *skip_pattern_storage; + } Vector grayscale_image = TRY(halftone_region.grayscale_image.visit( [](Vector const& grayscale_image) -> ErrorOr> { return grayscale_image; }, [&halftone_region, &pattern_dictionary](NonnullRefPtr const& reference) -> ErrorOr> { - // FIXME: This does not handle rotation or non-trivial grid vectors yet. - if (halftone_region.grid_offset_x_times_256 != 0 || halftone_region.grid_offset_y_times_256 != 0) - return Error::from_string_literal("JBIG2Writer: Halftone region match_image with non-zero grid offsets not yet implemented"); - if (pattern_dictionary.pattern_width != pattern_dictionary.pattern_height - || halftone_region.grid_vector_x_times_256 / 256 != pattern_dictionary.pattern_width - || halftone_region.grid_vector_y_times_256 != 0) - return Error::from_string_literal("JBIG2Writer: Halftone region match_image with non-trivial grid vectors not yet implemented"); - Vector converted_image; TRY(converted_image.try_resize(halftone_region.grayscale_width * halftone_region.grayscale_height)); - for (u32 y = 0; y < halftone_region.grayscale_height; ++y) { - for (u32 x = 0; x < halftone_region.grayscale_width; ++x) { + for (int m_g = 0; m_g < (int)halftone_region.grayscale_height; ++m_g) { + for (int n_g = 0; n_g < (int)halftone_region.grayscale_width; ++n_g) { + auto const x = (halftone_region.grid_offset_x_times_256 + m_g * halftone_region.grid_vector_y_times_256 + n_g * halftone_region.grid_vector_x_times_256) >> 8; + auto const y = (halftone_region.grid_offset_y_times_256 + m_g * halftone_region.grid_vector_x_times_256 - n_g * halftone_region.grid_vector_y_times_256) >> 8; + // Find best tile in pattern dictionary that matches reference best. // FIXME: This is a naive, inefficient implementation. u32 best_pattern_index = 0; @@ -2151,9 +2211,9 @@ static ErrorOr encode_halftone_region(JBIG2::HalftoneRegionSegmentData con u32 pattern_difference = 0; for (u32 py = 0; py < pattern_dictionary.pattern_height; ++py) { for (u32 px = 0; px < pattern_dictionary.pattern_width; ++px) { - int reference_x = x * pattern_dictionary.pattern_width + px; - int reference_y = y * pattern_dictionary.pattern_height + py; - if (reference_x >= reference->width() || reference_y >= reference->height()) + int reference_x = x + px; + int reference_y = y + py; + if (reference_x < 0 || reference_x >= reference->width() || reference_y < 0 || reference_y >= reference->height()) continue; auto pattern_pixel = pattern_dictionary.image->get_bit(pattern_x + px, py); auto reference_pixel = reference->get_pixel(reference_x, reference_y); @@ -2165,7 +2225,7 @@ static ErrorOr encode_halftone_region(JBIG2::HalftoneRegionSegmentData con best_pattern_index = pattern_index; } } - converted_image[y * halftone_region.grayscale_width + x] = best_pattern_index; + converted_image[m_g * halftone_region.grayscale_width + n_g] = best_pattern_index; } } @@ -2259,24 +2319,70 @@ static ErrorOr encode_generic_refinement_region(JBIG2::GenericRefinementRe // 7.4.7 Generic refinement region syntax if (header.referred_to_segments.size() > 1) return Error::from_string_literal("JBIG2Writer: Generic refinement region must refer to at most one segment"); - if (header.referred_to_segments.size() == 0) - return Error::from_string_literal("JBIG2Writer: Generic refinement region refining page not yet implemented"); - auto maybe_segment = context.segment_by_id.get(header.referred_to_segments[0].segment_number); - if (!maybe_segment.has_value()) - return Error::from_string_literal("JBIG2Writer: Could not find referred-to segment for generic refinement region"); - auto const& referred_to_segment = *maybe_segment.value(); + enum class CollectionBehavior { + ExcludeSelf, + IncludeSelf, + }; + auto collect_related_segments = [&context](u32 page, u32 segment_number, CollectionBehavior behavior) { + Vector preceding_segments_on_same_page; + bool found = false; + for (auto const& segment : context.segments) { + if (segment.header.page_association == 0 || segment.header.page_association == page) { + auto const& data = context.segment_data_by_id.get(segment.header.segment_number); + if (segment.header.segment_number == segment_number) { + if (behavior == CollectionBehavior::IncludeSelf) + preceding_segments_on_same_page.append(data->data); + found = true; + break; + } + preceding_segments_on_same_page.append(data->data); + } + } + VERIFY(found); + return preceding_segments_on_same_page; + }; - auto const* reference_bitmap = TRY(referred_to_segment.data.visit( - [](JBIG2::IntermediateGenericRegionSegmentData const& generic_region_wrapper) -> ErrorOr { - return generic_region_wrapper.generic_region.image; - }, - [](JBIG2::IntermediateGenericRefinementRegionSegmentData const& generic_refinement_region_wrapper) -> ErrorOr { - return generic_refinement_region_wrapper.generic_refinement_region.image; - }, - [](auto const&) -> ErrorOr { - return Error::from_string_literal("JBIG2Writer: Generic refinement region can only refer to intermediate region segments"); - })); + // 7.4.7.4 Reference bitmap selection + auto const reference_bitmap = TRY([&]() -> ErrorOr { + // "If this segment refers to another region segment, then set the reference bitmap GRREFERENCE to be the current + // contents of the auxiliary buffer associated with the region segment that this segment refers to." + if (header.referred_to_segments.size() == 1) { + auto maybe_segment = context.segment_by_id.get(header.referred_to_segments[0].segment_number); + if (!maybe_segment.has_value()) + return Error::from_string_literal("JBIG2Writer: Could not find referred-to segment for generic refinement region"); + auto const& referred_to_segment = *maybe_segment.value(); + + return TRY(referred_to_segment.data.visit( + [&] T>(T const&) -> ErrorOr> { + auto related_segments_on_same_page = collect_related_segments(referred_to_segment.header.page_association, referred_to_segment.header.segment_number, CollectionBehavior::IncludeSelf); + auto bitmap = TRY(JBIG2ImageDecoderPlugin::decode_embedded_intermediate_region_segment(related_segments_on_same_page, referred_to_segment.header.segment_number)); + return bitmap; + }, + + // Optimization: IntermediateGenericRegionSegmentData, IntermediateGenericRefinementRegionSegmentData could also use + // the implementation above, but since we happen to have the final bitmap at hand for them, just return it immediately. + [](JBIG2::IntermediateGenericRegionSegmentData const& generic_region_wrapper) -> ErrorOr> { + return generic_region_wrapper.generic_region.image; + }, + [](JBIG2::IntermediateGenericRefinementRegionSegmentData const& generic_refinement_region_wrapper) -> ErrorOr> { + return generic_refinement_region_wrapper.generic_refinement_region.image; + }, + + [](auto const&) -> ErrorOr> { + return Error::from_string_literal("JBIG2Writer: Generic refinement region can only refer to intermediate region segments"); + })) + ->as_subbitmap(); + } + + // "If this segment does not refer to another region segment, set GRREFERENCE to be a bitmap containing the current + // contents of the page buffer (see clause 8), restricted to the area of the page buffer specified by this segment's region + // segment information field." + VERIFY(header.referred_to_segments.size() == 0); + auto preceding_segments_on_same_page = collect_related_segments(header.page_association, header.segment_number, CollectionBehavior::ExcludeSelf); + auto bitmap = TRY(JBIG2ImageDecoderPlugin::decode_embedded(preceding_segments_on_same_page)); + return bitmap->subbitmap(generic_refinement_region.region_segment_information.rect()); + }()); GenericRefinementRegionEncodingInputParameters inputs { .image = *generic_refinement_region.image, @@ -2285,7 +2391,7 @@ static ErrorOr encode_generic_refinement_region(JBIG2::GenericRefinementRe inputs.gr_template = generic_refinement_region.flags & 1; inputs.is_typical_prediction_used = (generic_refinement_region.flags >> 1) & 1; inputs.adaptive_template_pixels = generic_refinement_region.adaptive_template_pixels; - RefinementContexts contexts { inputs.gr_template }; + JBIG2::RefinementContexts contexts { inputs.gr_template }; MQArithmeticEncoder encoder = TRY(MQArithmeticEncoder::initialize(0)); TRY(generic_refinement_region_encoding_procedure(inputs, encoder, contexts)); auto data = TRY(encoder.finalize(generic_refinement_region.trailing_7fff_handling)); @@ -2487,7 +2593,7 @@ static ErrorOr encode_extension(JBIG2::ExtensionData const& extension, Vec return {}; } -static ErrorOr encode_segment(Stream& stream, JBIG2::SegmentData const& segment_data, JBIG2EncodingContext& context) +static ErrorOr encode_segment(JBIG2::SegmentData const& segment_data, JBIG2EncodingContext& context) { Vector scratch_buffer; @@ -2504,6 +2610,10 @@ static ErrorOr encode_segment(Stream& stream, JBIG2::SegmentData const& se TRY(encode_text_region(text_region_wrapper.text_region, segment_data.header, context, scratch_buffer)); return scratch_buffer; }, + [&scratch_buffer, &segment_data, &context](JBIG2::IntermediateTextRegionSegmentData const& text_region_wrapper) -> ErrorOr { + TRY(encode_text_region(text_region_wrapper.text_region, segment_data.header, context, scratch_buffer)); + return scratch_buffer; + }, [&scratch_buffer](JBIG2::PatternDictionarySegmentData const& pattern_dictionary) -> ErrorOr { TRY(encode_pattern_dictionary(pattern_dictionary, scratch_buffer)); return scratch_buffer; @@ -2516,6 +2626,10 @@ static ErrorOr encode_segment(Stream& stream, JBIG2::SegmentData const& se TRY(encode_halftone_region(halftone_region_wrapper.halftone_region, segment_data.header, context, scratch_buffer)); return scratch_buffer; }, + [&scratch_buffer, &segment_data, &context](JBIG2::IntermediateHalftoneRegionSegmentData const& halftone_region_wrapper) -> ErrorOr { + TRY(encode_halftone_region(halftone_region_wrapper.halftone_region, segment_data.header, context, scratch_buffer)); + return scratch_buffer; + }, [&scratch_buffer](JBIG2::ImmediateGenericRegionSegmentData const& generic_region_wrapper) -> ErrorOr { TRY(encode_generic_region(generic_region_wrapper.generic_region, scratch_buffer)); return scratch_buffer; @@ -2573,9 +2687,11 @@ static ErrorOr encode_segment(Stream& stream, JBIG2::SegmentData const& se [](JBIG2::SymbolDictionarySegmentData const&) { return JBIG2::SegmentType::SymbolDictionary; }, [](JBIG2::ImmediateTextRegionSegmentData const&) { return JBIG2::SegmentType::ImmediateTextRegion; }, [](JBIG2::ImmediateLosslessTextRegionSegmentData const&) { return JBIG2::SegmentType::ImmediateLosslessTextRegion; }, + [](JBIG2::IntermediateTextRegionSegmentData const&) { return JBIG2::SegmentType::IntermediateTextRegion; }, [](JBIG2::PatternDictionarySegmentData const&) { return JBIG2::SegmentType::PatternDictionary; }, [](JBIG2::ImmediateHalftoneRegionSegmentData const&) { return JBIG2::SegmentType::ImmediateHalftoneRegion; }, [](JBIG2::ImmediateLosslessHalftoneRegionSegmentData const&) { return JBIG2::SegmentType::ImmediateLosslessHalftoneRegion; }, + [](JBIG2::IntermediateHalftoneRegionSegmentData const&) { return JBIG2::SegmentType::IntermediateHalftoneRegion; }, [](JBIG2::ImmediateGenericRegionSegmentData const&) { return JBIG2::SegmentType::ImmediateGenericRegion; }, [](JBIG2::ImmediateLosslessGenericRegionSegmentData const&) { return JBIG2::SegmentType::ImmediateLosslessGenericRegion; }, [](JBIG2::IntermediateGenericRegionSegmentData const&) { return JBIG2::SegmentType::IntermediateGenericRegion; }, @@ -2597,27 +2713,53 @@ static ErrorOr encode_segment(Stream& stream, JBIG2::SegmentData const& se header.data_length = segment_data.header.is_immediate_generic_region_of_initially_unknown_size ? 0xffff'ffff : encoded_data.size(); auto page_association_size = segment_data.header.force_32_bit_page_association ? PageAssociationSize::Force32Bit : PageAssociationSize::Auto; - TRY(encode_segment_header(stream, header, page_association_size)); - TRY(stream.write_until_depleted(encoded_data)); + AllocatingMemoryStream header_stream; + TRY(encode_segment_header(header_stream, header, page_association_size)); + auto header_data = TRY(header_stream.read_until_eof()); - return {}; + SerializedSegmentData data; + data.header_size = header_data.size(); + data.data = TRY(ByteBuffer::create_uninitialized(header_data.size() + encoded_data.size())); + header_data.span().copy_to(data.data.span()); + encoded_data.copy_to(data.data.span().slice(header_data.size())); + + return data; } ErrorOr JBIG2Writer::encode_with_explicit_data(Stream& stream, JBIG2::FileData const& file_data) { - if (file_data.header.organization != JBIG2::Organization::Sequential) - return Error::from_string_literal("JBIG2Writer: Can only encode sequential files yet"); + if (file_data.header.organization == JBIG2::Organization::Embedded) + return Error::from_string_literal("JBIG2Writer: Can only encode sequential or random-access files"); TRY(encode_jbig2_header(stream, file_data.header)); - JBIG2EncodingContext context; + JBIG2EncodingContext context { file_data.segments }; for (auto const& segment : file_data.segments) { if (context.segment_by_id.set(segment.header.segment_number, &segment) != HashSetResult::InsertedNewEntry) return Error::from_string_literal("JBIG2Writer: Duplicate segment number"); } for (auto const& segment : file_data.segments) { - TRY(encode_segment(stream, segment, context)); + auto data = TRY(encode_segment(segment, context)); + VERIFY(context.segment_data_by_id.set(segment.header.segment_number, data) == HashSetResult::InsertedNewEntry); + } + + if (file_data.header.organization == JBIG2::Organization::Sequential) { + for (auto const& segment : file_data.segments) { + auto const& data = context.segment_data_by_id.get(segment.header.segment_number); + TRY(stream.write_until_depleted(data->data)); + } + return {}; + } + + VERIFY(file_data.header.organization == JBIG2::Organization::RandomAccess); + for (auto const& segment : file_data.segments) { + auto const& data = context.segment_data_by_id.get(segment.header.segment_number); + TRY(stream.write_until_depleted(data->data.span().slice(0, data->header_size))); + } + for (auto const& segment : file_data.segments) { + auto const& data = context.segment_data_by_id.get(segment.header.segment_number); + TRY(stream.write_until_depleted(data->data.span().slice(data->header_size))); } return {}; diff --git a/Userland/Libraries/LibGfx/ImageFormats/JBIG2Writer.h b/Userland/Libraries/LibGfx/ImageFormats/JBIG2Writer.h index 6d0aaf9712bb12..e7c21a4db3152d 100644 --- a/Userland/Libraries/LibGfx/ImageFormats/JBIG2Writer.h +++ b/Userland/Libraries/LibGfx/ImageFormats/JBIG2Writer.h @@ -149,6 +149,10 @@ struct ImmediateLosslessTextRegionSegmentData { TextRegionSegmentData text_region; }; +struct IntermediateTextRegionSegmentData { + TextRegionSegmentData text_region; +}; + struct PatternDictionarySegmentData { u8 flags { 0 }; u8 pattern_width { 0 }; @@ -188,6 +192,10 @@ struct ImmediateLosslessHalftoneRegionSegmentData { HalftoneRegionSegmentData halftone_region; }; +struct IntermediateHalftoneRegionSegmentData { + HalftoneRegionSegmentData halftone_region; +}; + struct GenericRegionSegmentData { RegionSegmentInformationField region_segment_information {}; u8 flags { 0 }; @@ -263,9 +271,11 @@ struct SegmentData { SymbolDictionarySegmentData, ImmediateTextRegionSegmentData, ImmediateLosslessTextRegionSegmentData, + IntermediateTextRegionSegmentData, PatternDictionarySegmentData, ImmediateHalftoneRegionSegmentData, ImmediateLosslessHalftoneRegionSegmentData, + IntermediateHalftoneRegionSegmentData, ImmediateGenericRegionSegmentData, ImmediateLosslessGenericRegionSegmentData, IntermediateGenericRegionSegmentData, diff --git a/Userland/Libraries/LibGfx/ImageFormats/TIFFLoader.cpp b/Userland/Libraries/LibGfx/ImageFormats/TIFFLoader.cpp index e90756c39ead87..453309dfe219fe 100644 --- a/Userland/Libraries/LibGfx/ImageFormats/TIFFLoader.cpp +++ b/Userland/Libraries/LibGfx/ImageFormats/TIFFLoader.cpp @@ -589,8 +589,8 @@ class TIFFLoadingContext { ErrorOr set_next_ifd(u32 ifd_offset) { if (ifd_offset != 0) { - if (ifd_offset < TRY(m_stream->tell())) - return Error::from_string_literal("TIFFImageDecoderPlugin: Can not accept an IFD pointing to previous data"); + if (TRY(m_known_ifds.try_set(ifd_offset)) != HashSetResult::InsertedNewEntry) + return Error::from_string_literal("TIFFImageDecoderPlugin: Can not accept an IFD pointing to already visited data"); m_next_ifd = Optional { ifd_offset }; } else { @@ -773,6 +773,7 @@ class TIFFLoadingContext { RefPtr m_cmyk_bitmap {}; ByteOrder m_byte_order {}; + HashTable m_known_ifds {}; Optional m_next_ifd {}; ExifMetadata m_metadata {}; diff --git a/Userland/Libraries/LibGfx/ImageFormats/TIFFWriter.cpp b/Userland/Libraries/LibGfx/ImageFormats/TIFFWriter.cpp index 311cc1dcfeaaed..dcba07fa7ddf34 100644 --- a/Userland/Libraries/LibGfx/ImageFormats/TIFFWriter.cpp +++ b/Userland/Libraries/LibGfx/ImageFormats/TIFFWriter.cpp @@ -297,15 +297,7 @@ ErrorOr TIFFWriter::encode(Stream& stream, CMYKBitmap const& bitmap, Optio auto ifd_entries = make_cmyk_ifd(bitmap.size().width(), bitmap.size().height(), image_data_size, move(icc_data)); TRY(encode_ifd(stream, ifd_offset, move(ifd_entries))); - for (int y = 0; y < bitmap.size().height(); ++y) { - for (int x = 0; x < bitmap.size().width(); ++x) { - CMYK const& pixel = bitmap.scanline(y)[x]; - TRY(stream.write_value(pixel.c)); - TRY(stream.write_value(pixel.m)); - TRY(stream.write_value(pixel.y)); - TRY(stream.write_value(pixel.k)); - } - } + TRY(stream.write_until_depleted(bitmap.data())); return {}; } diff --git a/Userland/Libraries/LibHID/ReportDescriptorParser.cpp b/Userland/Libraries/LibHID/ReportDescriptorParser.cpp index 95d72278fa4830..7348117fe97005 100644 --- a/Userland/Libraries/LibHID/ReportDescriptorParser.cpp +++ b/Userland/Libraries/LibHID/ReportDescriptorParser.cpp @@ -367,7 +367,7 @@ ErrorOr ReportDescriptorParser::parse() break; case GlobalItemTag::Push: - m_item_state_table_stack.append(m_current_item_state_table); + TRY(m_item_state_table_stack.try_append(TRY(m_current_item_state_table.clone()))); break; case GlobalItemTag::Pop: @@ -464,7 +464,7 @@ ErrorOr ReportDescriptorParser::parse() } } - return m_parsed; + return move(m_parsed); } template @@ -511,15 +511,18 @@ ErrorOr ReportDescriptorParser::add_report_fields(FieldType field_type, It VERIFY_NOT_REACHED(); }(); - // FIXME: Since try_ensure does not return a reference to the contained value, we have to retrieve it separately. + // FIXME: Since try_ensure does not return a reference to the contained value, we have to implement it manually here. // This is a try_ensure bug that should be fixed. - (void)TRY(report_map.try_ensure(m_current_item_state_table.global.report_id.value_or(0), [this] { - return Report { - .size_in_bits = static_cast(m_parsed.uses_report_ids ? 8 : 0), - .fields = {}, - }; - })); - auto& report = report_map.get(m_current_item_state_table.global.report_id.value_or(0)).release_value(); + auto report_id = m_current_item_state_table.global.report_id.value_or(0); + if (report_map.find(report_id) == report_map.end()) { + auto result = TRY(report_map.try_set(report_id, + Report { + .size_in_bits = static_cast(m_parsed.uses_report_ids ? 8 : 0), + .fields = {}, + })); + VERIFY(result == HashSetResult::InsertedNewEntry); + } + auto& report = report_map.get(report_id).release_value(); size_t const field_size_in_bits = m_current_item_state_table.global.report_size.value(); diff --git a/Userland/Libraries/LibHID/ReportDescriptorParser.h b/Userland/Libraries/LibHID/ReportDescriptorParser.h index 308a7cec9fc1fa..100f206b4ccccf 100644 --- a/Userland/Libraries/LibHID/ReportDescriptorParser.h +++ b/Userland/Libraries/LibHID/ReportDescriptorParser.h @@ -87,6 +87,24 @@ ErrorOr dump_report_descriptor(ReadonlyBytes); // 5.4 Item Parser struct ItemStateTable { + ErrorOr clone() const + { + return ItemStateTable { + .global = global, + .local = { + .usages = TRY(local.usages.clone()), + .usage_minimum = local.usage_minimum, + .usage_maximum = local.usage_maximum, + .designator_index = local.designator_index, + .degignator_minimum = local.degignator_minimum, + .designator_maximum = local.designator_maximum, + .string_index = local.string_index, + .string_minimum = local.string_minimum, + .string_maximum = local.string_maximum, + }, + }; + } + // 6.2.2.7 Global Items struct { Optional usage_page; diff --git a/Userland/Libraries/LibLine/Editor.cpp b/Userland/Libraries/LibLine/Editor.cpp index 45465aea20acce..9e122f435fb76d 100644 --- a/Userland/Libraries/LibLine/Editor.cpp +++ b/Userland/Libraries/LibLine/Editor.cpp @@ -1940,7 +1940,6 @@ StringMetrics Editor::actual_rendered_string_metrics(Utf32View const& view, RedB for (size_t break_index = 0; break_index < grapheme_breaks.size(); ++break_index) { auto i = grapheme_breaks[break_index]; - auto c = view[i]; if (!mask_it.is_end() && mask_it.key() <= i) mask = *mask_it; @@ -1950,8 +1949,20 @@ StringMetrics Editor::actual_rendered_string_metrics(Utf32View const& view, RedB continue; } + auto next_grapheme_start = break_index + 1 < grapheme_breaks.size() ? grapheme_breaks[break_index + 1] : view.length(); auto next_c = break_index + 1 < grapheme_breaks.size() ? view.code_points()[grapheme_breaks[break_index + 1]] : 0; + auto c = view[i]; state = actual_rendered_string_length_step(metrics, i, current_line, c, next_c, state, mask, maximum_line_width, last_return); + + for (size_t j = i + 1; j < next_grapheme_start; ++j) { + // Consume the rest of the code points in this grapheme cluster without updating the state; this is just to account for their length properly. + current_line.length++; + current_line.visible_length++; + metrics.total_length++; + if (current_line.bit_length.has_value()) + current_line.bit_length.value() += code_point_length_in_utf8(view[j]); + } + if (!mask_it.is_end() && mask_it.key() <= i) { auto mask_it_peek = mask_it; ++mask_it_peek; diff --git a/Userland/Libraries/LibPDF/DocumentParser.cpp b/Userland/Libraries/LibPDF/DocumentParser.cpp index 4b048d6bd8a2c5..53fcb3c83f55cf 100644 --- a/Userland/Libraries/LibPDF/DocumentParser.cpp +++ b/Userland/Libraries/LibPDF/DocumentParser.cpp @@ -183,7 +183,7 @@ PDFErrorOr DocumentParser::initialize_linea auto dict_value = indirect_value_or_error.value()->value(); if (!dict_value.has>()) - return error("Expected linearization object to be a dictionary"); + return LinearizationResult::NotLinearized; auto dict_object = dict_value.get>(); if (!dict_object->is()) diff --git a/Userland/Libraries/LibPDF/Filter.cpp b/Userland/Libraries/LibPDF/Filter.cpp index a64c37302caf3b..9641b2e9a2297e 100644 --- a/Userland/Libraries/LibPDF/Filter.cpp +++ b/Userland/Libraries/LibPDF/Filter.cpp @@ -10,6 +10,7 @@ #include #include #include +#include #include #include #include @@ -309,7 +310,7 @@ PDFErrorOr Filter::decode_jbig2(Document* document, ReadonlyBytes by } segments.append(bytes); - auto decoded = TRY(Gfx::JBIG2ImageDecoderPlugin::decode_embedded(segments)); + auto decoded = TRY(TRY(Gfx::JBIG2ImageDecoderPlugin::decode_embedded(segments))->to_byte_buffer()); // JBIG2 treats `1` as "ink present" (black) and `0` as "no ink" (white). // PDF treats `1` as "light present" (white) and `1` as "no light" (black). diff --git a/Userland/Libraries/LibPartition/EBRPartitionTable.cpp b/Userland/Libraries/LibPartition/EBRPartitionTable.cpp index a459c1ad9387d8..f866d93ab0539c 100644 --- a/Userland/Libraries/LibPartition/EBRPartitionTable.cpp +++ b/Userland/Libraries/LibPartition/EBRPartitionTable.cpp @@ -29,7 +29,7 @@ void EBRPartitionTable::search_extended_partition(MBRPartitionTable& checked_ebr // If we are pointed to an invalid logical partition, something is seriously wrong. VERIFY(checked_logical_partition.has_value()); - m_partitions.append(checked_logical_partition.value().offset(current_block_offset)); + m_partitions.try_append(checked_logical_partition.value().offset(current_block_offset)).release_value_but_fixme_should_propagate_errors(); if (!checked_ebr.contains_ebr()) return; current_block_offset += checked_ebr.partition(1).value().start_block(); diff --git a/Userland/Libraries/LibPartition/GUIDPartitionTable.cpp b/Userland/Libraries/LibPartition/GUIDPartitionTable.cpp index 7ff39051c42a10..e28c932dbcd9f3 100644 --- a/Userland/Libraries/LibPartition/GUIDPartitionTable.cpp +++ b/Userland/Libraries/LibPartition/GUIDPartitionTable.cpp @@ -108,7 +108,7 @@ bool GUIDPartitionTable::initialize() Array unique_guid {}; unique_guid.span().overwrite(0, entry.unique_guid, unique_guid.size()); dbgln("Detected GPT partition (entry={}), offset={}, limit={}", entry_index, entry.first_lba, entry.last_lba); - m_partitions.append({ entry.first_lba, entry.last_lba, partition_type, unique_guid, entry.attributes }); + m_partitions.try_append({ entry.first_lba, entry.last_lba, partition_type, unique_guid, entry.attributes }).release_value_but_fixme_should_propagate_errors(); raw_byte_index += header().partition_entry_size; } diff --git a/Userland/Libraries/LibPartition/PartitionTable.h b/Userland/Libraries/LibPartition/PartitionTable.h index e35be0cae2b14e..1a9602df00154a 100644 --- a/Userland/Libraries/LibPartition/PartitionTable.h +++ b/Userland/Libraries/LibPartition/PartitionTable.h @@ -19,7 +19,7 @@ class PartitionTable { virtual ~PartitionTable() = default; virtual bool is_valid() const = 0; - Vector partitions() const { return m_partitions; } + Vector const& partitions() const { return m_partitions; } size_t block_size() const { return m_device.block_size(); } protected: diff --git a/Userland/Libraries/LibRegex/RegexMatcher.h b/Userland/Libraries/LibRegex/RegexMatcher.h index 666a4d6710973f..49b874d6ed8f7b 100644 --- a/Userland/Libraries/LibRegex/RegexMatcher.h +++ b/Userland/Libraries/LibRegex/RegexMatcher.h @@ -130,19 +130,24 @@ class Regex final { start_offset = match.global_offset + match.view.length(); GenericLexer lexer(replacement_pattern); while (!lexer.is_eof()) { - if (lexer.consume_specific('\\')) { - if (lexer.consume_specific('\\')) { - builder.append('\\'); - continue; - } + if (lexer.consume_specific('&')) { + builder.append(result.matches[i].view.to_byte_string()); + } else if (!lexer.consume_specific('\\')) { + builder.append(lexer.consume_while([](auto ch) { return ch != '\\' && ch != '&'; })); + } else if (lexer.consume_specific('&')) { + builder.append('&'); + } else if (lexer.consume_specific('\\')) { + builder.append('\\'); + } else { auto number = lexer.consume_while(isdigit); - if (auto index = number.to_number(); index.has_value() && result.n_capture_groups >= index.value()) { - builder.append(result.capture_group_matches[i][index.value() - 1].view.to_byte_string()); - } else { + auto index = number.to_number(); + if (!index.has_value() || result.n_capture_groups < index.value()) { builder.appendff("\\{}", number); + } else if (index.value() == 0) { + builder.append(result.matches[i].view.to_byte_string()); + } else { + builder.append(result.capture_group_matches[i][index.value() - 1].view.to_byte_string()); } - } else { - builder.append(lexer.consume_while([](auto ch) { return ch != '\\'; })); } } } diff --git a/Userland/Libraries/LibVT/EscapeSequenceParser.cpp b/Userland/Libraries/LibVT/EscapeSequenceParser.cpp index 249b5478bc2d7b..fa978d3902ceb5 100644 --- a/Userland/Libraries/LibVT/EscapeSequenceParser.cpp +++ b/Userland/Libraries/LibVT/EscapeSequenceParser.cpp @@ -26,7 +26,7 @@ Vector EscapeSequenceParser::osc_parameters( // This should not be a problem as we won't dereference the 0-length Span that's created. // Using &m_osc_raw[prev_idx] to get the start pointer checks whether we're out of bounds, // so we would crash. - params.append({ m_osc_raw.data() + prev_idx, end_idx - prev_idx }); + params.try_append({ m_osc_raw.data() + prev_idx, end_idx - prev_idx }).release_value_but_fixme_should_propagate_errors(); prev_idx = end_idx; } return params; @@ -57,7 +57,7 @@ void EscapeSequenceParser::perform_action(EscapeSequenceStateMachine::Action act if (m_param_vector.size() == MAX_PARAMETERS) m_ignoring = true; else - m_param_vector.append(m_param); + m_param_vector.try_append(m_param).release_value_but_fixme_should_propagate_errors(); m_executor.dcs_hook(m_param_vector, intermediates(), m_ignoring, byte); break; case EscapeSequenceStateMachine::Action::Put: @@ -91,17 +91,17 @@ void EscapeSequenceParser::perform_action(EscapeSequenceStateMachine::Action act if (m_osc_parameter_indexes.size() == MAX_OSC_PARAMETERS) { dbgln("EscapeSequenceParser::perform_action: shenanigans! OSC sequence has too many parameters"); } else { - m_osc_parameter_indexes.append(m_osc_raw.size()); + m_osc_parameter_indexes.try_append(m_osc_raw.size()).release_value_but_fixme_should_propagate_errors(); } } else { - m_osc_raw.append(byte); + m_osc_raw.try_append(byte).release_value_but_fixme_should_propagate_errors(); } break; case EscapeSequenceStateMachine::Action::OscEnd: if (m_osc_parameter_indexes.size() == MAX_OSC_PARAMETERS) { dbgln("EscapeSequenceParser::perform_action: shenanigans! OSC sequence has too many parameters"); } else { - m_osc_parameter_indexes.append(m_osc_raw.size()); + m_osc_parameter_indexes.try_append(m_osc_raw.size()).release_value_but_fixme_should_propagate_errors(); } m_executor.execute_osc_sequence(osc_parameters(), byte); break; @@ -113,7 +113,7 @@ void EscapeSequenceParser::perform_action(EscapeSequenceStateMachine::Action act dbgln("EscapeSequenceParser::perform_action: shenanigans! CSI sequence has too many parameters"); m_ignoring = true; } else { - m_param_vector.append(m_param); + m_param_vector.try_append(m_param).release_value_but_fixme_should_propagate_errors(); } m_executor.execute_csi_sequence(m_param_vector, intermediates(), m_ignoring, byte); @@ -136,7 +136,7 @@ void EscapeSequenceParser::perform_action(EscapeSequenceStateMachine::Action act m_ignoring = true; } else { if (byte == ';') { - m_param_vector.append(m_param); + m_param_vector.try_append(m_param).release_value_but_fixme_should_propagate_errors(); m_param = 0; } else if (byte == ':') { dbgln("EscapeSequenceParser::perform_action: subparameters are not yet implemented"); diff --git a/Userland/Libraries/LibVT/Line.cpp b/Userland/Libraries/LibVT/Line.cpp index 6c09c0056b861e..6947a3cc282a4d 100644 --- a/Userland/Libraries/LibVT/Line.cpp +++ b/Userland/Libraries/LibVT/Line.cpp @@ -35,7 +35,7 @@ void Line::rewrap(size_t new_length, Line* next_line, CursorPosition* cursor, bo void Line::set_length(size_t new_length) { - m_cells.resize(new_length); + m_cells.try_resize(new_length).release_value_but_fixme_should_propagate_errors(); if (m_terminated_at.has_value()) m_terminated_at = min(*m_terminated_at, new_length); } diff --git a/Userland/Libraries/LibVT/TerminalWidget.cpp b/Userland/Libraries/LibVT/TerminalWidget.cpp index d048767094a587..ea53d4de9d0e46 100644 --- a/Userland/Libraries/LibVT/TerminalWidget.cpp +++ b/Userland/Libraries/LibVT/TerminalWidget.cpp @@ -479,6 +479,11 @@ void TerminalWidget::paint_event(GUI::PaintEvent& event) void TerminalWidget::set_window_progress(int value, int max) { + if (max == 0) { + window()->set_progress(OptionalNone {}); + return; + } + float float_value = value; float float_max = max; float progress = (float_value / float_max) * 100.0f; diff --git a/Userland/Libraries/LibWeb/HTML/HTMLInputElement.cpp b/Userland/Libraries/LibWeb/HTML/HTMLInputElement.cpp index dd152461d8003d..285f9b45ea82dc 100644 --- a/Userland/Libraries/LibWeb/HTML/HTMLInputElement.cpp +++ b/Userland/Libraries/LibWeb/HTML/HTMLInputElement.cpp @@ -2151,7 +2151,7 @@ WebIDL::ExceptionOr HTMLInputElement::set_value_as_number(double value) return WebIDL::InvalidStateError::create(realm(), "valueAsNumber: Invalid input type used"_string); // Otherwise, if the new value is a Not-a-Number (NaN) value, then set the value of the element to the empty string. - if (value == NAN) { + if (value == AK::NaN) { TRY(set_value(String {})); return {}; } diff --git a/Userland/Libraries/LibWeb/Painting/PaintableBox.cpp b/Userland/Libraries/LibWeb/Painting/PaintableBox.cpp index 7a025683b014ee..6bb28fc12a60e6 100644 --- a/Userland/Libraries/LibWeb/Painting/PaintableBox.cpp +++ b/Userland/Libraries/LibWeb/Painting/PaintableBox.cpp @@ -857,7 +857,7 @@ bool PaintableBox::handle_mousewheel(Badge, CSSPixelPoint, unsigne if (!layout_box().is_user_scrollable()) return false; - // TODO: Vertical and horizontal scroll overflow should be handled seperately. + // TODO: Vertical and horizontal scroll overflow should be handled separately. if (!has_scrollable_overflow()) return false; diff --git a/Userland/Services/CrashDaemon/main.cpp b/Userland/Services/CrashDaemon/main.cpp index ae8dbbaa290b44..e16165dfd46652 100644 --- a/Userland/Services/CrashDaemon/main.cpp +++ b/Userland/Services/CrashDaemon/main.cpp @@ -4,32 +4,11 @@ * SPDX-License-Identifier: BSD-2-Clause */ -#include -#include #include #include #include #include #include -#include -#include -#include -#include - -static void wait_until_coredump_is_ready(ByteString const& coredump_path) -{ - while (true) { - struct stat statbuf; - if (stat(coredump_path.characters(), &statbuf) < 0) { - perror("stat"); - VERIFY_NOT_REACHED(); - } - if (statbuf.st_mode & 0400) // Check if readable - break; - - usleep(10000); // sleep for 10ms - } -} static void launch_crash_reporter(ByteString const& coredump_path, bool unlink_on_exit) { @@ -49,13 +28,14 @@ ErrorOr serenity_main(Main::Arguments) TRY(watcher.add_watch("/tmp/coredump", Core::FileWatcherEvent::Type::ChildCreated)); while (true) { - auto event = watcher.wait_for_event(); - VERIFY(event.has_value()); - if (event.value().type != Core::FileWatcherEvent::Type::ChildCreated) + auto event = watcher.wait_for_event().value(); + if (event.type != Core::FileWatcherEvent::Type::ChildCreated) continue; - auto& coredump_path = event.value().event_path; + auto& coredump_path = event.event_path; + if (coredump_path.ends_with(".partial"sv)) + continue; + dbgln("New coredump file: {}", coredump_path); - wait_until_coredump_is_ready(coredump_path); auto file_or_error = Core::MappedFile::map(coredump_path); if (file_or_error.is_error()) { diff --git a/Userland/Services/SpiceAgent/FileTransferOperation.h b/Userland/Services/SpiceAgent/FileTransferOperation.h index 2e0d7b1079d5fd..6a168d4710a27c 100644 --- a/Userland/Services/SpiceAgent/FileTransferOperation.h +++ b/Userland/Services/SpiceAgent/FileTransferOperation.h @@ -36,7 +36,7 @@ class FileTransferOperation : public RefCounted { // Fired by SpiceAgent when we have received all of the data needed for this transfer. ErrorOr complete_transfer(SpiceAgent& agent); - // Fired by the SpiceAgent when it recieves data related to this transfer. + // Fired by the SpiceAgent when it receives data related to this transfer. ErrorOr on_data_received(FileTransferDataMessage& message); private: diff --git a/Userland/Services/SpiceAgent/Message.h b/Userland/Services/SpiceAgent/Message.h index ed3c70fe60b5ea..d0c4d41c394d6c 100644 --- a/Userland/Services/SpiceAgent/Message.h +++ b/Userland/Services/SpiceAgent/Message.h @@ -225,7 +225,7 @@ class FileTransferStartMessage : public Message { Metadata m_metadata; }; -// Sent/recieved to indicate the status of the current file transfer. +// Sent/received to indicate the status of the current file transfer. class FileTransferStatusMessage : public Message { public: FileTransferStatusMessage(u32 id, FileTransferStatus status) diff --git a/Userland/Services/SpiceAgent/SpiceAgent.h b/Userland/Services/SpiceAgent/SpiceAgent.h index a58fd00dd7f4c4..43947ae004bc3d 100644 --- a/Userland/Services/SpiceAgent/SpiceAgent.h +++ b/Userland/Services/SpiceAgent/SpiceAgent.h @@ -20,7 +20,7 @@ namespace SpiceAgent { // The maximum amount of data that can be contained within a message's buffer. -// If the buffer's length is equal to this, then the next data recieved will be more data from the same buffer. +// If the buffer's length is equal to this, then the next data received will be more data from the same buffer. constexpr u32 message_buffer_threshold = 2048; // The maximum amount of data that can be received in one file transfer message diff --git a/Userland/Services/SystemServer/Service.cpp b/Userland/Services/SystemServer/Service.cpp index cde6d0810da2a5..a5e215dfee5717 100644 --- a/Userland/Services/SystemServer/Service.cpp +++ b/Userland/Services/SystemServer/Service.cpp @@ -66,6 +66,8 @@ ErrorOr Service::setup_socket(SocketDescriptor& socket) auto un = un_optional.value(); TRY(Core::System::bind(socket_fd, (sockaddr const*)&un, sizeof(un))); + socket.was_created = true; + TRY(Core::System::listen(socket_fd, 16)); return {}; } @@ -346,7 +348,7 @@ Service::Service(Core::ConfigFile const& config, StringView name) // be applied for every socket. mode_t permissions = strtol(socket_perms.at(min(socket_perms.size() - 1, (long unsigned)i)).characters(), nullptr, 8) & 0777; - m_sockets.empend(path.value(), -1, permissions); + m_sockets.empend(path.value(), -1, permissions, false); } } @@ -381,7 +383,11 @@ ErrorOr Service::determine_account(int fd) Service::~Service() { for (auto& socket : m_sockets) { - if (auto rc = remove(socket.path.characters()); rc != 0) - dbgln("{}", Error::from_errno(errno)); + if (!socket.was_created) + continue; + + auto result = Core::System::unlink(socket.path); + if (result.is_error()) + dbgln("unlink(\"{}\") failed: {}", socket.path, result.error()); } } diff --git a/Userland/Services/SystemServer/Service.h b/Userland/Services/SystemServer/Service.h index f1c3f7650bf4cf..d915d582a6718e 100644 --- a/Userland/Services/SystemServer/Service.h +++ b/Userland/Services/SystemServer/Service.h @@ -47,6 +47,8 @@ class Service final : public Core::EventReceiver { int fd { -1 }; /// File permissions of the socket. mode_t permissions; + /// Set to true if the socket file was successfully created. + bool was_created; }; // Path to the executable. By default this is /bin/{m_name}. diff --git a/Userland/Services/SystemServer/main.cpp b/Userland/Services/SystemServer/main.cpp index bf1af5b382743b..53fc44768638ba 100644 --- a/Userland/Services/SystemServer/main.cpp +++ b/Userland/Services/SystemServer/main.cpp @@ -18,7 +18,9 @@ #include #include #include +#include #include +#include #include #include #include @@ -124,20 +126,35 @@ static ErrorOr activate_services(Core::ConfigFile const& config) static ErrorOr activate_base_services_based_on_system_mode() { if (g_system_mode == graphical_system_mode) { - bool found_gpu_device = false; - for (int attempt = 0; attempt < 10; attempt++) { - struct stat file_state; - int rc = lstat("/dev/gpu/connector0", &file_state); - if (rc == 0) { - found_gpu_device = true; - break; - } - sleep(1); - } - if (!found_gpu_device) { + bool done_searching_for_gpu = false; + + auto timeout = Core::Timer::create_single_shot(10000, [&]() { dbgln("WARNING: No device nodes at /dev/gpu/ directory after 10 seconds. This is probably a sign of disabled graphics functionality."); dbgln("To cope with this, graphical mode will not be enabled."); g_system_mode = text_system_mode; + + done_searching_for_gpu = true; + }); + + auto log = Core::Timer::create_single_shot(1000, [&]() { + dbgln("Waiting on /dev/gpu/connector0 to appear..."); + }); + + auto watcher = TRY(Core::FileWatcher::create()); + watcher->on_change = [&](Core::FileWatcherEvent const& event) { + if (event.event_path != "/dev/gpu/connector0"sv) + return; + done_searching_for_gpu = true; + }; + + TRY(watcher->add_watch("/dev/gpu/", Core::FileWatcherEvent::Type::ChildCreated)); + + // The GPU might have appeared while we were setting up the watcher. + // Only wait for the file if we can't stat it. + if (Core::System::lstat("/dev/gpu/connector0"sv).is_error()) { + log->start(); + timeout->start(); + Core::EventLoop::current().spin_until([&]() { return done_searching_for_gpu; }); } } diff --git a/Userland/Utilities/CMakeLists.txt b/Userland/Utilities/CMakeLists.txt index 42d70b679eaf0d..41da01fe4216f1 100644 --- a/Userland/Utilities/CMakeLists.txt +++ b/Userland/Utilities/CMakeLists.txt @@ -359,7 +359,6 @@ target_link_libraries(paste PRIVATE LibGUI) target_link_libraries(patch PRIVATE LibDiff LibFileSystem) target_link_libraries(pdf PRIVATE LibGfx LibPDF) target_link_libraries(pgrep PRIVATE LibRegex) -target_link_libraries(ping PRIVATE LibCrypto) target_link_libraries(pixelflut PRIVATE LibImageDecoderClient LibIPC LibGfx) target_link_libraries(pkill PRIVATE LibRegex) target_link_libraries(pledge PRIVATE LibELF) @@ -384,7 +383,6 @@ target_link_libraries(test-imap PRIVATE LibIMAP) target_link_libraries(test-jpeg-roundtrip PRIVATE LibGfx) target_link_libraries(test-pthread PRIVATE LibThreading) target_link_libraries(touch PRIVATE LibFileSystem) -target_link_libraries(traceroute PRIVATE LibCrypto) target_link_libraries(unzip PRIVATE LibArchive LibCompress LibCrypto LibFileSystem) target_link_libraries(update-cpp-test-results PRIVATE LibCpp) target_link_libraries(useradd PRIVATE LibCrypt) diff --git a/Userland/Utilities/animation.cpp b/Userland/Utilities/animation.cpp index 5b351e660aba1a..08d4b0cb739cf6 100644 --- a/Userland/Utilities/animation.cpp +++ b/Userland/Utilities/animation.cpp @@ -14,17 +14,18 @@ #include struct Options { - StringView in_path; + Vector in_paths; StringView out_path; bool write_full_frames { false }; Gfx::AnimationWriter::AllowInterFrameCompression allow_inter_frame_compression { Gfx::AnimationWriter::AllowInterFrameCompression::Yes }; + Optional frame_duration_ms; }; static ErrorOr parse_options(Main::Arguments arguments) { Options options; Core::ArgsParser args_parser; - args_parser.add_positional_argument(options.in_path, "Path to input image file", "FILE"); + args_parser.add_positional_argument(options.in_paths, "Paths to input image file", "FILE"); args_parser.add_option(options.out_path, "Path to output image file", "output", 'o', "FILE"); bool inter_frame_compression_full = false; @@ -36,6 +37,8 @@ static ErrorOr parse_options(Main::Arguments arguments) bool inter_frame_compression_none = false; args_parser.add_option(inter_frame_compression_none, "Do not store incremental frames. Produces larger files.", "inter-frame-compression=none"); + args_parser.add_option(options.frame_duration_ms, "Frame duration in ms (default: from input)", "frame-duration-ms", {}, {}); + args_parser.parse(arguments); if (options.out_path.is_empty()) @@ -55,34 +58,54 @@ ErrorOr serenity_main(Main::Arguments arguments) { Options options = TRY(parse_options(arguments)); - // FIXME: Allow multiple single frames as input too, and allow manually setting their duration. + if (options.in_paths.is_empty()) + return Error::from_string_literal("Need at least one input file"); + + Vector> files; + Vector> decoders; + for (auto in_path : options.in_paths) { + files.append(TRY(Core::MappedFile::map(in_path))); + auto decoder = TRY(Gfx::ImageDecoder::try_create_for_raw_bytes(files.last()->bytes())); + if (!decoder) + return Error::from_string_literal("Could not find decoder for input file"); + decoders.append(decoder.release_nonnull()); + } + + VERIFY(!decoders.is_empty()); + auto output_size = decoders[0]->size(); + for (auto const& decoder : decoders) { + if (decoder->size() != output_size) + return Error::from_string_literal("All input images must have the same dimensions"); + } - auto file = TRY(Core::MappedFile::map(options.in_path)); - auto decoder = TRY(Gfx::ImageDecoder::try_create_for_raw_bytes(file->bytes())); - if (!decoder) - return Error::from_string_literal("Could not find decoder for input file"); + // FIXME: Make overridable? + auto output_loop_count = decoders[0]->loop_count(); auto output_file = TRY(Core::File::open(options.out_path, Core::File::OpenMode::Write)); auto output_stream = TRY(Core::OutputBufferedFile::create(move(output_file))); auto animation_writer = TRY([&]() -> ErrorOr> { if (options.out_path.ends_with(".apng"sv)) - return Gfx::PNGWriter::start_encoding_animation(*output_stream, decoder->size(), decoder->loop_count()); + return Gfx::PNGWriter::start_encoding_animation(*output_stream, output_size, output_loop_count); if (options.out_path.ends_with(".webp"sv)) - return Gfx::WebPWriter::start_encoding_animation(*output_stream, decoder->size(), decoder->loop_count()); + return Gfx::WebPWriter::start_encoding_animation(*output_stream, output_size, output_loop_count); if (options.out_path.ends_with(".gif"sv)) - return Gfx::GIFWriter::start_encoding_animation(*output_stream, decoder->size(), decoder->loop_count()); + return Gfx::GIFWriter::start_encoding_animation(*output_stream, output_size, output_loop_count); return Error::from_string_literal("Unable to find a encoder for the requested extension."); }()); RefPtr last_frame; - for (size_t i = 0; i < decoder->frame_count(); ++i) { - auto frame = TRY(decoder->frame(i)); - if (options.write_full_frames) { - TRY(animation_writer->add_frame(*frame.image, frame.duration)); - } else { - TRY(animation_writer->add_frame_relative_to_last_frame(*frame.image, frame.duration, last_frame, options.allow_inter_frame_compression)); - last_frame = frame.image; + for (auto const& decoder : decoders) { + for (size_t i = 0; i < decoder->frame_count(); ++i) { + auto frame = TRY(decoder->frame(i)); + auto frame_duration = options.frame_duration_ms.value_or(frame.duration); + + if (options.write_full_frames) { + TRY(animation_writer->add_frame(*frame.image, frame_duration)); + } else { + TRY(animation_writer->add_frame_relative_to_last_frame(*frame.image, frame_duration, last_frame, options.allow_inter_frame_compression)); + last_frame = frame.image; + } } } diff --git a/Userland/Utilities/config.cpp b/Userland/Utilities/config.cpp index 28fc9091778675..b9c5c215e46c82 100644 --- a/Userland/Utilities/config.cpp +++ b/Userland/Utilities/config.cpp @@ -4,11 +4,25 @@ * SPDX-License-Identifier: BSD-2-Clause */ +#include #include #include #include #include +static void print_group(StringView domain, StringView group) +{ + auto keys = Config::Client::the().list_keys(domain, group); + quick_sort(keys); + for (auto const& key : keys) { + auto value = Config::Client::the().read_string_value(domain, group, key); + if (!value.has_value()) + warnln("Can't find a value for {}:{}:{}", domain, group, key); + else + outln("{}={}", key, *value); + } +} + ErrorOr serenity_main(Main::Arguments arguments) { Core::EventLoop loop; @@ -22,26 +36,39 @@ ErrorOr serenity_main(Main::Arguments arguments) args_parser.set_general_help("Show or modify values in the configuration files through ConfigServer."); args_parser.add_option(remove, "Remove group or key", "remove", 'r'); args_parser.add_positional_argument(domain, "Config domain", "domain"); - args_parser.add_positional_argument(group, "Group name", "group"); + args_parser.add_positional_argument(group, "Group name", "group", Core::ArgsParser::Required::No); args_parser.add_positional_argument(key, "Key name", "key", Core::ArgsParser::Required::No); args_parser.add_positional_argument(value_to_write, "Value to write", "value", Core::ArgsParser::Required::No); args_parser.parse(arguments); if (remove) { - if (!key.is_empty()) - Config::remove_key(domain, group, key); - else + if (group.is_empty()) + return Error::from_string_literal("Can't delete a domain"); + if (key.is_empty()) Config::remove_group(domain, group); + else + Config::remove_key(domain, group, key); return 0; } - if (key.is_empty() && value_to_write.is_empty()) { - Config::add_group(domain, group); + if (!group.is_empty() && !key.is_empty() && !value_to_write.is_empty()) { + Config::write_string(domain, group, key, value_to_write); return 0; } - if (!key.is_empty() && !value_to_write.is_empty()) { - Config::write_string(domain, group, key, value_to_write); + if (group.is_empty()) { + auto groups = Config::Client::the().list_groups(domain); + quick_sort(groups); + + for (auto const& group : groups) { + outln("[{}]", group); + print_group(domain, group); + } + return 0; + } + + if (key.is_empty()) { + print_group(domain, group); return 0; } diff --git a/Userland/Utilities/icc.cpp b/Userland/Utilities/icc.cpp index 20329a260213c3..2fde45d099f428 100644 --- a/Userland/Utilities/icc.cpp +++ b/Userland/Utilities/icc.cpp @@ -1,10 +1,12 @@ /* - * Copyright (c) 2022-2023, Nico Weber + * Copyright (c) 2022-2025, Nico Weber * * SPDX-License-Identifier: BSD-2-Clause */ #include +#include +#include #include #include #include @@ -228,21 +230,27 @@ static ErrorOr print_profile_measurement(Gfx::ICC::Profile const& profile) return {}; } -static ErrorOr print_color_as_sRGB(StringView color, Gfx::ICC::Profile const& profile) +template +static ErrorOr> split_color_string(StringView color, Gfx::ICC::ColorSpace color_space) { auto split = color.split_view(','); - if (number_of_components_in_color_space(profile.data_color_space()) != split.size()) - return Error::from_string_literal("unexpected number of color in color string"); + if (number_of_components_in_color_space(color_space) != split.size()) + return Error::from_string_literal("unexpected number of colors in color string"); - Vector channels; - for (auto [i, channel] : enumerate(split)) { - auto maybe_number = channel.to_number(); + Vector channels; + for (auto channel : split) { + auto maybe_number = channel.to_number(); if (!maybe_number.has_value()) return Error::from_string_literal("unable to parse color string"); channels.append(*maybe_number); } + return channels; +} +static ErrorOr print_color_as_sRGB(StringView color, Gfx::ICC::Profile const& profile) +{ + auto channels = TRY(split_color_string(color, profile.data_color_space())); auto srgb_profile = TRY(Gfx::ICC::sRGB()); auto pcs = TRY(profile.to_pcs(channels)); Array out_color_buffer {}; @@ -253,6 +261,72 @@ static ErrorOr print_color_as_sRGB(StringView color, Gfx::ICC::Profile con return {}; } +static ErrorOr print_stdin_u8_to_pcs(Gfx::ICC::Profile const& profile) +{ + auto stdin = TRY(Core::InputBufferedFile::create(TRY(Core::File::standard_input()))); + ByteBuffer buffer = TRY(ByteBuffer::create_uninitialized(1024)); + while (!stdin->is_eof()) { + auto line = TRY(stdin->read_line(buffer)); + if (line.is_empty()) + continue; + auto channels = TRY(split_color_string(line, profile.data_color_space())); + auto pcs = TRY(profile.to_pcs(channels)); + if (profile.connection_space() == Gfx::ICC::ColorSpace::PCSLAB) { + outln("pcslab({}, {}, {})", pcs[0], pcs[1], pcs[2]); + } else if (profile.connection_space() == Gfx::ICC::ColorSpace::PCSXYZ) { + outln("pcsxyz({}, {}, {})", pcs[0], pcs[1], pcs[2]); + } else { + VERIFY_NOT_REACHED(); + } + } + return {}; +} + +static ErrorOr print_stdin_u8_from_pcs(Gfx::ICC::Profile const& profile) +{ + auto stdin = TRY(Core::InputBufferedFile::create(TRY(Core::File::standard_input()))); + ByteBuffer buffer = TRY(ByteBuffer::create_uninitialized(1024)); + while (!stdin->is_eof()) { + auto line = TRY(stdin->read_line(buffer)); + if (line.is_empty()) + continue; + + GenericLexer lexer(line); + Gfx::ICC::ColorSpace pcs_space = Gfx::ICC::ColorSpace::FifteenColor; + if (lexer.consume_specific("pcslab")) + pcs_space = Gfx::ICC::ColorSpace::PCSLAB; + else if (lexer.consume_specific("pcsxyz")) + pcs_space = Gfx::ICC::ColorSpace::PCSXYZ; + else + return Error::from_string_literal("expected 'pcslab' or 'pcsxyz'"); + lexer.consume_while([](char c) { return is_ascii_space(c); }); + if (!lexer.consume_specific('(')) + return Error::from_string_literal("expected '(' after color space"); + StringView remaining = lexer.remaining(); + if (!remaining.ends_with(')')) + return Error::from_string_literal("expected ')' at end of color"); + remaining = remaining.substring_view(0, remaining.length() - 1); + auto pcs_vector = TRY(split_color_string(remaining, pcs_space)); + FloatVector3 pcs { pcs_vector[0], pcs_vector[1], pcs_vector[2] }; + + Vector out_colors; + out_colors.resize(number_of_components_in_color_space(profile.data_color_space())); + + // It's fishy to pass profile.pcs_illuminant() here; we really have to pass the illuminant of the input PCS color, + // but we don't have that information. But at least the v4 spec dictates that the PCS illuminant always is + // D50 (X = 0,9642, Y = 1,0 and Z = 0,8249), and ICC::Profile's parse_pcs_illuminant() rejects any non-D50 + // (except for v2 with D65, which it emits a warning for and then treats as D50), so for every profile we can + // load, pcs_illuminant() is D50 here and this will do the right thing. + TRY(profile.from_pcs(pcs_space, profile.pcs_illuminant(), pcs, out_colors)); + + out("{}", out_colors[0]); + for (size_t i = 1; i < out_colors.size(); ++i) + out(", {}", out_colors[i]); + outln(); + } + return {}; +} + ErrorOr serenity_main(Main::Arguments arguments) { Core::ArgsParser args_parser; @@ -261,7 +335,7 @@ ErrorOr serenity_main(Main::Arguments arguments) args_parser.add_positional_argument(path, "Path to ICC profile or to image containing ICC profile", "FILE", Core::ArgsParser::Required::No); StringView name; - args_parser.add_option(name, "Name of a built-in profile, such as 'sRGB'", "name", 'n', "NAME"); + args_parser.add_option(name, "Name of a built-in profile, such as 'sRGB', 'LAB', 'XYZ'", "name", 'n', "NAME"); StringView dump_out_path; args_parser.add_option(dump_out_path, "Dump unmodified ICC profile bytes to this path", "dump-to", 0, "FILE"); @@ -278,6 +352,12 @@ ErrorOr serenity_main(Main::Arguments arguments) StringView color_to_convert; args_parser.add_option(color_to_convert, "Convert a color from the given profile to sRGB", "color-to-sRGB", 0, "c1, c2, c3[, c4]"); + bool stdin_u8_to_pcs = false; + args_parser.add_option(stdin_u8_to_pcs, "Read colors from stdin and print PCS to stdout", "stdin-u8-to-pcs"); + + bool stdin_u8_from_pcs = false; + args_parser.add_option(stdin_u8_from_pcs, "Read PCS colors from stdin and print profile colors to stdout", "stdin-u8-from-pcs"); + bool force_print = false; args_parser.add_option(force_print, "Print profile even when writing ICC files", "print"); @@ -299,8 +379,24 @@ ErrorOr serenity_main(Main::Arguments arguments) ReadonlyBytes icc_bytes; NonnullRefPtr profile = TRY([&]() -> ErrorOr> { if (!name.is_empty()) { + if (name == "LAB") + return Gfx::ICC::IdentityLAB(); + if (name == "LAB_mft2") + return Gfx::ICC::IdentityLAB_mft2(); + if (name == "LAB_mABmBA_no_clut") + return Gfx::ICC::IdentityLAB_mABmBA_no_clut(); + if (name == "LAB_mABmBA_u8_clut") + return Gfx::ICC::IdentityLAB_mABmBA_u8_clut(); + if (name == "LAB_mABmBA_u16_clut") + return Gfx::ICC::IdentityLAB_mABmBA_u16_clut(); + if (name == "XYZ_mABmBA_no_clut") + return Gfx::ICC::IdentityXYZ_D50_mABmBA_no_clut(); + if (name == "XYZ_mABmBA_u16_clut") + return Gfx::ICC::IdentityXYZ_D50_mABmBA_u16_clut(); if (name == "sRGB") return Gfx::ICC::sRGB(); + if (name == "XYZ") + return Gfx::ICC::IdentityXYZ_D50(); return Error::from_string_literal("unknown profile name"); } auto file = TRY(Core::MappedFile::map(path)); @@ -348,6 +444,16 @@ ErrorOr serenity_main(Main::Arguments arguments) return 0; } + if (stdin_u8_to_pcs) { + TRY(print_stdin_u8_to_pcs(*profile)); + return 0; + } + + if (stdin_u8_from_pcs) { + TRY(print_stdin_u8_from_pcs(*profile)); + return 0; + } + bool do_print = (dump_out_path.is_empty() && reencode_out_path.is_empty() && !measure) || force_print; if (!do_print) return 0; diff --git a/Userland/Utilities/image.cpp b/Userland/Utilities/image.cpp index 0aeaecced91a65..1ea0fd0529b3c2 100644 --- a/Userland/Utilities/image.cpp +++ b/Userland/Utilities/image.cpp @@ -169,23 +169,37 @@ static ErrorOr convert_image_profile(LoadedImage& image, StringView conver auto source_profile = TRY(Gfx::ICC::Profile::try_load_from_externally_owned_memory(source_icc_data)); auto destination_profile = TRY(Gfx::ICC::Profile::try_load_from_externally_owned_memory(*image.icc_data)); - if (destination_profile->data_color_space() != Gfx::ICC::ColorSpace::RGB) - return Error::from_string_literal("Can only convert to RGB at the moment, but destination color space is not RGB"); + if (destination_profile->data_color_space() != Gfx::ICC::ColorSpace::RGB + && destination_profile->data_color_space() != Gfx::ICC::ColorSpace::CMYK) + return Error::from_string_literal("Can only convert to RGB and CMYK at the moment, but destination color space is neither"); if (image.bitmap.has>()) { if (source_profile->data_color_space() != Gfx::ICC::ColorSpace::CMYK) return Error::from_string_literal("Source image data is CMYK but source color space is not CMYK"); - auto& cmyk_frame = image.bitmap.get>(); - auto rgb_frame = TRY(Gfx::Bitmap::create(Gfx::BitmapFormat::BGRx8888, cmyk_frame->size())); - TRY(destination_profile->convert_cmyk_image(*rgb_frame, *cmyk_frame, *source_profile)); - image.bitmap = RefPtr(move(rgb_frame)); - image.internal_format = Gfx::NaturalFrameFormat::RGB; + if (destination_profile->data_color_space() == Gfx::ICC::ColorSpace::CMYK) { + auto& cmyk_frame = image.bitmap.get>(); + TRY(destination_profile->convert_cmyk_image_to_cmyk_image(*cmyk_frame, *source_profile)); + } else { + auto& cmyk_frame = image.bitmap.get>(); + auto rgb_frame = TRY(Gfx::Bitmap::create(Gfx::BitmapFormat::BGRx8888, cmyk_frame->size())); + TRY(destination_profile->convert_cmyk_image(*rgb_frame, *cmyk_frame, *source_profile)); + image.bitmap = RefPtr(move(rgb_frame)); + image.internal_format = Gfx::NaturalFrameFormat::RGB; + } } else { // FIXME: This likely wrong for grayscale images because they've been converted to // RGB at this point, but their embedded color profile is still for grayscale. - auto& frame = image.bitmap.get>(); - TRY(destination_profile->convert_image(*frame, *source_profile)); + if (destination_profile->data_color_space() == Gfx::ICC::ColorSpace::CMYK) { + auto& rgb_frame = image.bitmap.get>(); + auto cmyk_frame = TRY(Gfx::CMYKBitmap::create_with_size(rgb_frame->size())); + TRY(destination_profile->convert_image_to_cmyk_image(*cmyk_frame, *rgb_frame, *source_profile)); + image.bitmap = RefPtr(move(cmyk_frame)); + image.internal_format = Gfx::NaturalFrameFormat::CMYK; + } else { + auto& frame = image.bitmap.get>(); + TRY(destination_profile->convert_image(*frame, *source_profile)); + } } return {}; diff --git a/Userland/Utilities/imgcmp.cpp b/Userland/Utilities/imgcmp.cpp index 849398189ecc13..696614a126dc7b 100644 --- a/Userland/Utilities/imgcmp.cpp +++ b/Userland/Utilities/imgcmp.cpp @@ -65,8 +65,8 @@ ErrorOr serenity_main(Main::Arguments arguments) StringView write_diff_image_path; args_parser.add_option(write_diff_image_path, "Write image that highlights differing pixels", "write-diff-image", {}, "FILE"); - bool should_count = false; - args_parser.add_option(should_count, "Report the number of differing pixels", "count", {}); + bool quiet = false; + args_parser.add_option(quiet, "Only set exit code, print no output", "quiet", {}); StringView first_image_path; args_parser.add_positional_argument(first_image_path, "Path to first input image", "FILE1"); @@ -90,21 +90,60 @@ ErrorOr serenity_main(Main::Arguments arguments) } u64 number_of_differences = 0; + int first_different_x = 0; + int first_different_y = 0; + u8 max_error_r = 0; + u8 max_error_g = 0; + u8 max_error_b = 0; + u8 max_error = 0; + int max_error_x = 0; + int max_error_y = 0; + u64 total_error_r = 0; + u64 total_error_g = 0; + u64 total_error_b = 0; for (int y = 0; y < first_image->physical_height(); ++y) { for (int x = 0; x < first_image->physical_width(); ++x) { auto first_pixel = first_image->get_pixel(x, y); auto second_pixel = second_image->get_pixel(x, y); if (first_pixel != second_pixel) { - if (number_of_differences == 0) - warnln("different pixel at ({}, {}), {} vs {}", x, y, first_pixel, second_pixel); - if (!should_count) + if (quiet) return 1; + if (number_of_differences == 0) { + first_different_x = x; + first_different_y = y; + } + auto error_r = abs((int)first_pixel.red() - (int)second_pixel.red()); + auto error_g = abs((int)first_pixel.green() - (int)second_pixel.green()); + auto error_b = abs((int)first_pixel.blue() - (int)second_pixel.blue()); + max_error_r = max(max_error_r, error_r); + max_error_g = max(max_error_g, error_g); + max_error_b = max(max_error_b, error_b); + auto pixel_max_error = max(max(error_r, error_g), error_b); + if (pixel_max_error > max_error) { + max_error = pixel_max_error; + max_error_x = x; + max_error_y = y; + } + total_error_r += error_r; + total_error_g += error_g; + total_error_b += error_b; number_of_differences++; } } } - if (should_count) - warnln("number of differing pixels: {}", number_of_differences); + if (!quiet && number_of_differences > 0) { + u64 number_of_pixels = first_image->physical_width() * first_image->physical_height(); + warnln("number of differing pixels: {} ({:.2f}%)", number_of_differences, (100.0 * number_of_differences) / number_of_pixels); + warnln("max error R: {:4}, G: {:4}, B: {:4}", max_error_r, max_error_g, max_error_b); + warnln("avg error R: {:.2f}, G: {:.2f}, B: {:.2f}", + (double)total_error_r / number_of_pixels, + (double)total_error_g / number_of_pixels, + (double)total_error_b / number_of_pixels); + warnln("max error at ({}, {}): {} vs {}", max_error_x, max_error_y, + first_image->get_pixel(max_error_x, max_error_y), second_image->get_pixel(max_error_x, max_error_y)); + warnln("first difference at ({}, {}): {} vs {}", first_different_x, first_different_y, + first_image->get_pixel(first_different_x, first_different_y), second_image->get_pixel(first_different_x, first_different_y)); + } - return 0; + return number_of_differences > 0 ? 1 : 0; } diff --git a/Userland/Utilities/jbig2-from-json.cpp b/Userland/Utilities/jbig2-from-json.cpp index 73c314ca439903..839da0c2611129 100644 --- a/Userland/Utilities/jbig2-from-json.cpp +++ b/Userland/Utilities/jbig2-from-json.cpp @@ -187,7 +187,7 @@ static ErrorOr jbig2_rect_from_json(JsonObject const& object) return rect; } -static ErrorOr> jbig2_bitmap_from_json(ToJSONOptions const& options, ByteString const& base_name) +static ErrorOr> jbig2_load_bitmap(ToJSONOptions const& options, ByteString const& base_name) { RefPtr bitmap; @@ -206,16 +206,60 @@ static ErrorOr> jbig2_bitmap_from_json(ToJSONOptions return TRY(decoder->frame(0)).image.release_nonnull(); } +static ErrorOr> jbig2_bitmap_from_json(ToJSONOptions const& options, JsonObject const& object) +{ + RefPtr bitmap; + JSONRect crop_rect; + + TRY(object.try_for_each_member([&](StringView key, JsonValue const& value) -> ErrorOr { + if (key == "from_file") { + if (value.is_string()) { + bitmap = TRY(jbig2_load_bitmap(options, value.as_string())); + return {}; + } + return Error::from_string_literal("expected string for \"from_file\""); + } + + if (key == "crop") { + if (value.is_object()) { + crop_rect = TRY(jbig2_rect_from_json(value.as_object())); + return {}; + } + return Error::from_string_literal("expected object for \"crop\""); + } + + dbgln("match_image key {}", key); + return Error::from_string_literal("unknown match_image key"); + })); + + if (!bitmap) + return Error::from_string_literal("no image data in match_image; add \"from_file\" key"); + + if (crop_rect.x.has_value() || crop_rect.y.has_value() || crop_rect.width.has_value() || crop_rect.height.has_value()) { + int crop_x = static_cast(crop_rect.x.value_or(0)); + int crop_y = static_cast(crop_rect.y.value_or(0)); + int crop_width = static_cast(crop_rect.width.value_or(bitmap->width() - crop_x)); + int crop_height = static_cast(crop_rect.height.value_or(bitmap->height() - crop_y)); + if (crop_x + crop_width > bitmap->width() || crop_y + crop_height > bitmap->height()) + return Error::from_string_literal("crop rectangle out of bounds"); + bitmap = TRY(bitmap->cropped({ crop_x, crop_y, crop_width, crop_height })); + } + + return bitmap.release_nonnull(); +} + static ErrorOr> jbig2_image_from_json(ToJSONOptions const& options, JsonObject const& object) { RefPtr image; JSONRect crop_rect; bool invert = false; + int repeat_x = 1; + int repeat_y = 1; TRY(object.try_for_each_member([&](StringView key, JsonValue const& value) -> ErrorOr { if (key == "from_file") { if (value.is_string()) { - auto bitmap = TRY(jbig2_bitmap_from_json(options, value.as_string())); + auto bitmap = TRY(jbig2_load_bitmap(options, value.as_string())); image = TRY(Gfx::BilevelImage::create_from_bitmap(*bitmap, Gfx::DitheringAlgorithm::FloydSteinberg)); return {}; } @@ -238,6 +282,22 @@ static ErrorOr> jbig2_image_from_json(ToJSONOpt return Error::from_string_literal("expected bool for \"invert\""); } + if (key == "repeat_x") { + if (auto repeat_x_value = value.get_i32(); repeat_x_value.has_value() && repeat_x_value.value() >= 1) { + repeat_x = repeat_x_value.value(); + return {}; + } + return Error::from_string_literal("expected i32 >= 1 for \"repeat_x\""); + } + + if (key == "repeat_y") { + if (auto repeat_y_value = value.get_i32(); repeat_y_value.has_value() && repeat_y_value.value() >= 1) { + repeat_y = repeat_y_value.value(); + return {}; + } + return Error::from_string_literal("expected i32 >= 1 for \"repeat_y\""); + } + dbgln("image_data key {}", key); return Error::from_string_literal("unknown image_data key"); })); @@ -261,6 +321,16 @@ static ErrorOr> jbig2_image_from_json(ToJSONOpt image = move(cropped_image); } + if (repeat_x > 1 || repeat_y > 1) { + auto repeated_image = TRY(Gfx::BilevelImage::create(image->width() * repeat_x, image->height() * repeat_y)); + for (u32 y = 0; y < repeated_image->height(); ++y) { + for (u32 x = 0; x < repeated_image->width(); ++x) { + repeated_image->set_bit(x, y, image->get_bit(x % image->width(), y % image->height())); + } + } + image = move(repeated_image); + } + if (invert) { for (u32 y = 0; y < image->height(); ++y) for (u32 x = 0; x < image->width(); ++x) @@ -460,7 +530,7 @@ static ErrorOr jbig2_symbol_dictionary_flags_from_json(JsonObject const& ob flags |= 1u << 9; return {}; } - return Error::from_string_literal("expected bool for \"is_bitmap_coding_context_used\""); + return Error::from_string_literal("expected bool for \"is_bitmap_coding_context_retained\""); } if (key == "template"sv) { @@ -1324,6 +1394,11 @@ static ErrorOr jbig2_immediate_lossless_text_region_fro return Gfx::JBIG2::SegmentData { header, Gfx::JBIG2::ImmediateLosslessTextRegionSegmentData { TRY(jbig2_text_region_from_json(options, object)) } }; } +static ErrorOr jbig2_intermediate_text_region_from_json(ToJSONOptions const& options, Gfx::JBIG2::SegmentHeaderData const& header, Optional object) +{ + return Gfx::JBIG2::SegmentData { header, Gfx::JBIG2::IntermediateTextRegionSegmentData { TRY(jbig2_text_region_from_json(options, object)) } }; +} + static ErrorOr jbig2_pattern_dictionary_flags_from_json(JsonObject const& object) { u8 flags = 0; @@ -1373,6 +1448,12 @@ static ErrorOr jbig2_pattern_dictionary_from_json(ToJSO UniqueImageTiles, }; Method method = Method::None; + u32 grayscale_width { 0 }; + u32 grayscale_height { 0 }; + i32 grid_offset_x_times_256 { 0 }; + i32 grid_offset_y_times_256 { 0 }; + u16 grid_vector_x_times_256 { 0 }; + u16 grid_vector_y_times_256 { 0 }; TRY(object->try_for_each_member([&](StringView key, JsonValue const& value) -> ErrorOr { if (key == "flags"sv) { @@ -1422,13 +1503,82 @@ static ErrorOr jbig2_pattern_dictionary_from_json(ToJSO return {}; } + if (key == "grayscale_width"sv) { + if (auto grayscale_width_json = value.get_u32(); grayscale_width_json.has_value()) { + grayscale_width = grayscale_width_json.value(); + return {}; + } + return Error::from_string_literal("expected u32 for \"grayscale_width\""); + } + + if (key == "grayscale_height"sv) { + if (auto grayscale_height_json = value.get_u32(); grayscale_height_json.has_value()) { + grayscale_height = grayscale_height_json.value(); + return {}; + } + return Error::from_string_literal("expected u32 for \"grayscale_height\""); + } + + if (key == "grid_offset_x_times_256"sv) { + if (auto grid_offset_x_times_256_json = value.get_i32(); grid_offset_x_times_256_json.has_value()) { + grid_offset_x_times_256 = grid_offset_x_times_256_json.value(); + return {}; + } + return Error::from_string_literal("expected i32 for \"grid_offset_x_times_256\""); + } + + if (key == "grid_offset_y_times_256"sv) { + if (auto grid_offset_y_times_256_json = value.get_i32(); grid_offset_y_times_256_json.has_value()) { + grid_offset_y_times_256 = grid_offset_y_times_256_json.value(); + return {}; + } + return Error::from_string_literal("expected i32 for \"grid_offset_y_times_256\""); + } + + if (key == "grid_vector_x_times_256"sv) { + if (auto grid_vector_x_times_256_json = value.get_u32(); grid_vector_x_times_256_json.has_value()) { + if (grid_vector_x_times_256_json.value() > 0xffff) + return Error::from_string_literal("expected u16 for \"grid_vector_x_times_256\""); + grid_vector_x_times_256 = grid_vector_x_times_256_json.value(); + return {}; + } + return Error::from_string_literal("expected u16 for \"grid_vector_x_times_256\""); + } + + if (key == "grid_vector_y_times_256"sv) { + if (auto grid_vector_y_times_256_json = value.get_u32(); grid_vector_y_times_256_json.has_value()) { + if (grid_vector_y_times_256_json.value() > 0xffff) + return Error::from_string_literal("expected u16 for \"grid_vector_y_times_256\""); + grid_vector_y_times_256 = grid_vector_y_times_256_json.value(); + return {}; + } + return Error::from_string_literal("expected u16 for \"grid_vector_y_times_256\""); + } + // FIXME: Make this more flexible. if (key == "image_data"sv) { if (value.is_object()) { image = TRY(jbig2_image_from_json(options, value.as_object())); return {}; } - return Error::from_string_literal("expected object for \"image_data\""); + if (value.is_array()) { + size_t width = 0; + for (auto const& [i, image_json] : enumerate(value.as_array().values())) { + if (!image_json.is_object()) + return Error::from_string_literal("expected object for \"image_data\" array entries"); + auto tile_image = TRY(jbig2_image_from_json(options, image_json.as_object())); + if (i == 0) { + width = tile_image->width(); + image = TRY(Gfx::BilevelImage::create(width * value.as_array().size(), tile_image->height())); + } + if (tile_image->width() != width || tile_image->height() != image->height()) + return Error::from_string_literal("all images in \"image_data\" array must have the same dimensions"); + Gfx::IntPoint destination_position { static_cast(i * width), 0 }; + tile_image->composite_onto(*image, destination_position, Gfx::BilevelImage::CompositionType::Replace); + } + return {}; + } + return Error::from_string_literal("expected object or array for \"image_data\""); } if (key == "method"sv) { @@ -1443,7 +1593,7 @@ static ErrorOr jbig2_pattern_dictionary_from_json(ToJSO return {}; } } - return Error::from_string_literal("expected \"distinct_image_tiles\" for \"method\""); + return Error::from_string_literal("expected \"distinct_image_tiles\" or \"unique_image_tiles\" for \"method\""); } dbgln("pattern_dictionary key {}", key); @@ -1453,20 +1603,36 @@ static ErrorOr jbig2_pattern_dictionary_from_json(ToJSO if (gray_max_from_tiles && method == Method::None) return Error::from_string_literal("can't use \"from_tiles\" for gray_max without using a tiling method"); + if (!image) + return Error::from_string_literal("pattern_dictionary \"data\" object missing \"image_data\""); + if (method == Method::DistinctImageTiles || method == Method::UniqueImageTiles) { - auto number_of_tiles_in_x = ceil_div(image->width(), static_cast(pattern_width)); - auto number_of_tiles_in_y = ceil_div(image->height(), static_cast(pattern_height)); + if (grid_vector_x_times_256 == 0 && grid_vector_y_times_256 == 0) { + if (grayscale_width == 0 && grayscale_height == 0) { + grayscale_width = ceil_div(image->width(), static_cast(pattern_width)); + grayscale_height = ceil_div(image->height(), static_cast(pattern_height)); + } + grid_vector_x_times_256 = pattern_width * 256; + } + + if (grayscale_width == 0 || grayscale_height == 0) + return Error::from_string_literal("grayscale_width and grayscale_height must be set when using custom grid"); // FIXME: For UniqueImageTiles at the edge, we could use a custom hasher/comparator to match existing full tiles // by ignoring pixels outside the clipped tile rect. Vector tiles; HashTable saw_tile; Gfx::IntRect bitmap_rect { 0, 0, static_cast(image->width()), static_cast(image->height()) }; - for (size_t tile_y = 0, tile_index = 0; tile_y < number_of_tiles_in_y; ++tile_y) { - for (size_t tile_x = 0; tile_x < number_of_tiles_in_x; ++tile_x, ++tile_index) { - Gfx::IntPoint source_position { static_cast(tile_x * pattern_width), static_cast(tile_y * pattern_height) }; + for (int tile_y = 0; tile_y < (int)grayscale_height; ++tile_y) { + for (int tile_x = 0; tile_x < (int)grayscale_width; ++tile_x) { + auto x = (grid_offset_x_times_256 + tile_y * grid_vector_y_times_256 + tile_x * grid_vector_x_times_256) >> 8; + auto y = (grid_offset_y_times_256 + tile_y * grid_vector_x_times_256 - tile_x * grid_vector_y_times_256) >> 8; + + Gfx::IntPoint source_position { x, y }; Gfx::IntRect source_rect { source_position, { pattern_width, pattern_height } }; source_rect = source_rect.intersected(bitmap_rect); + if (source_rect.is_empty()) + continue; auto source = image->subbitmap(source_rect); if (method == Method::DistinctImageTiles || saw_tile.set(source) == HashSetResult::InsertedNewEntry) TRY(tiles.try_append(source)); @@ -1476,6 +1642,9 @@ static ErrorOr jbig2_pattern_dictionary_from_json(ToJSO auto tiled_image = TRY(Gfx::BilevelImage::create(pattern_width * tiles.size(), pattern_height)); tiled_image->fill(false); for (auto const& [i, tile] : enumerate(tiles)) { + // FIXME: The destination_position is wrong for tiles clipped at the left or top edge. + // We should remember the original source_position shift after intersection with bitmap_rect, + // and add that offset here. Gfx::IntPoint destination_position { static_cast(i * pattern_width), 0 }; tile.composite_onto(*tiled_image, destination_position, Gfx::BilevelImage::CompositionType::Replace); } @@ -1484,6 +1653,10 @@ static ErrorOr jbig2_pattern_dictionary_from_json(ToJSO gray_max = tiles.size() - 1; image = move(tiled_image); + } else if (grayscale_width != 0 || grayscale_height != 0 + || grid_offset_x_times_256 != 0 || grid_offset_y_times_256 != 0 + || grid_vector_x_times_256 != 0 || grid_vector_y_times_256 != 0) { + return Error::from_string_literal("grid parameters ignored when \"method\" is not set to a tiling method"); } return Gfx::JBIG2::SegmentData { @@ -1605,11 +1778,15 @@ static ErrorOr, NonnullRefPtr>> jbig2_halftone_ } if (key == "match_image") { + if (value.is_object()) { + graymap = TRY(jbig2_bitmap_from_json(options, value.as_object())); + return {}; + } if (value.is_string()) { - graymap = TRY(jbig2_bitmap_from_json(options, value.as_string())); + graymap = TRY(jbig2_load_bitmap(options, value.as_string())); return {}; } - return Error::from_string_literal("expected string for \"match_image\""); + return Error::from_string_literal("expected string or object for \"match_image\""); } dbgln("graymap_data key {}", key); @@ -1764,6 +1941,11 @@ static ErrorOr jbig2_immediate_lossless_halftone_region return Gfx::JBIG2::SegmentData { header, Gfx::JBIG2::ImmediateLosslessHalftoneRegionSegmentData { TRY(jbig2_halftone_region_from_json(options, object)) } }; } +static ErrorOr jbig2_intermediate_halftone_region_from_json(ToJSONOptions const& options, Gfx::JBIG2::SegmentHeaderData const& header, Optional object) +{ + return Gfx::JBIG2::SegmentData { header, Gfx::JBIG2::IntermediateHalftoneRegionSegmentData { TRY(jbig2_halftone_region_from_json(options, object)) } }; +} + static ErrorOr jbig2_generic_region_flags_from_json(JsonObject const& object) { u8 flags = 0; @@ -2664,12 +2846,16 @@ static ErrorOr jbig2_segment_from_json(ToJSONOptions co return jbig2_immediate_text_region_from_json(options, header, segment_data_object); if (type_string == "lossless_text_region") return jbig2_immediate_lossless_text_region_from_json(options, header, segment_data_object); + if (type_string == "intermediate_text_region") + return jbig2_intermediate_text_region_from_json(options, header, segment_data_object); if (type_string == "pattern_dictionary") return jbig2_pattern_dictionary_from_json(options, header, segment_data_object); if (type_string == "halftone_region") return jbig2_immediate_halftone_region_from_json(options, header, segment_data_object); if (type_string == "lossless_halftone_region") return jbig2_immediate_lossless_halftone_region_from_json(options, header, segment_data_object); + if (type_string == "intermediate_halftone_region") + return jbig2_intermediate_halftone_region_from_json(options, header, segment_data_object); if (type_string == "generic_region") return jbig2_immediate_generic_region_from_json(options, header, segment_data_object); if (type_string == "lossless_generic_region") @@ -2763,6 +2949,7 @@ ErrorOr serenity_main(Main::Arguments arguments) // Only write images that decode correctly. Gfx::JBIG2DecoderOptions decoder_options; decoder_options.log_comments = Gfx::JBIG2DecoderOptions::LogComments::No; + decoder_options.strictness = Gfx::JBIG2DecoderOptions::Strictness::SpecCompliant; TRY(TRY(Gfx::JBIG2ImageDecoderPlugin::create_with_options(jbig2_data, decoder_options))->frame(0)); auto output_stream = TRY(Core::File::open(out_path, Core::File::OpenMode::Write)); diff --git a/Userland/Utilities/ping.cpp b/Userland/Utilities/ping.cpp index f76706e06e388f..24007977fce747 100644 --- a/Userland/Utilities/ping.cpp +++ b/Userland/Utilities/ping.cpp @@ -6,9 +6,9 @@ #include #include +#include #include #include -#include #include #include #include @@ -212,7 +212,7 @@ ErrorOr serenity_main(Main::Arguments arguments) ping_packet[i + sizeof(struct icmphdr)] = i & 0xFF; } - ping_hdr->checksum = Crypto::Checksum::IPv4Header({ ping_packet.data(), ping_packet.size() }).digest(); + ping_hdr->checksum = bit_cast(InternetChecksum({ ping_packet.data(), ping_packet.size() }).digest()); struct timeval tv_send; gettimeofday(&tv_send, nullptr); diff --git a/Userland/Utilities/run-tests.cpp b/Userland/Utilities/run-tests.cpp index 4f7fd5aad82e32..d2fade922c98f1 100644 --- a/Userland/Utilities/run-tests.cpp +++ b/Userland/Utilities/run-tests.cpp @@ -185,6 +185,8 @@ void TestRunner::do_run_single_test(ByteString const& test_path, size_t current_ out(" {}", test_relative_path); print_modifiers({ Test::CLEAR, Test::ITALIC, Test::FG_GRAY }); + if (test_result.child_pid != 0) + out(" - PID ({}) -", test_result.child_pid); if (test_result.time_taken < 1000) { outln(" ({}ms)", static_cast(test_result.time_taken)); } else { diff --git a/Userland/Utilities/sed.cpp b/Userland/Utilities/sed.cpp index 5387210af22816..5588e910b9c0b6 100644 --- a/Userland/Utilities/sed.cpp +++ b/Userland/Utilities/sed.cpp @@ -242,13 +242,13 @@ struct RArguments : FilepathArgument { }; struct SArguments { - Regex regex; + Variant, Regex> regex; StringView replacement; PosixOptions options; bool print; Optional output_filepath; - static SedErrorOr parse(GenericLexer& lexer) + static SedErrorOr parse(GenericLexer& lexer, bool strict_posix) { auto generic_error_message = "Incomplete substitution command"sv; @@ -307,7 +307,11 @@ struct SArguments { } } - return SArguments { Regex { pattern }, replacement, options, print, output_filepath }; + if (strict_posix) { + return SArguments { Regex { pattern }, replacement, options, print, output_filepath }; + } else { + return SArguments { Regex { pattern }, replacement, options, print, output_filepath }; + } } private: @@ -503,7 +507,7 @@ static SedErrorOr verify_number_of_addresses(Command const& command) return {}; } -static SedErrorOr parse_command(GenericLexer& lexer) +static SedErrorOr parse_command(GenericLexer& lexer, bool strict_posix) { lexer.consume_while(is_ascii_blank); @@ -551,7 +555,7 @@ static SedErrorOr parse_command(GenericLexer& lexer) command.arguments = TRY(RArguments::parse(lexer)); break; case 's': - command.arguments = TRY(SArguments::parse(lexer)); + command.arguments = TRY(SArguments::parse(lexer, strict_posix)); break; case 't': command.arguments = TRY(TArguments::parse(lexer)); @@ -586,7 +590,7 @@ static SedErrorOr parse_command(GenericLexer& lexer) class Script { public: - [[nodiscard]] bool add_script_part(StringView data) + [[nodiscard]] bool add_script_part(StringView data, bool strict_posix) { auto last_pos = m_script.length(); m_script.append(data); @@ -594,7 +598,7 @@ class Script { while (!lexer.is_eof()) { if (lexer.is_eof()) break; - auto maybe_command = parse_command(lexer); + auto maybe_command = parse_command(lexer, strict_posix); if (maybe_command.is_error()) { warnln("Problem while parsing script part: {}", maybe_command.release_error().message()); return false; @@ -866,7 +870,7 @@ static ErrorOr apply(Command const& command, StringBuilder& patte case 's': { auto pattern_space_sv = pattern_space.string_view(); auto const& s_args = command.arguments->get(); - auto result = s_args.regex.replace(pattern_space_sv, s_args.replacement, s_args.options); + auto result = s_args.regex.visit([&](auto& re) { return re.replace(pattern_space_sv, s_args.replacement, s_args.options); }); auto replacement_made = result != pattern_space_sv; pattern_space.clear(); pattern_space.append(result); @@ -1001,9 +1005,11 @@ ErrorOr serenity_main(Main::Arguments args) bool suppress_default_output = false; bool edit_in_place = false; + bool strict_posix = false; Core::ArgsParser arg_parser; Script script; Vector pos_args; + Vector script_parts; arg_parser.set_general_help("The Stream EDitor"); arg_parser.add_option(suppress_default_output, "suppress default output", nullptr, 'n'); arg_parser.add_option(Core::ArgsParser::Option { @@ -1011,7 +1017,7 @@ ErrorOr serenity_main(Main::Arguments args) .help_string = "A file containing script commands", .short_name = 'f', .value_name = "script-file", - .accept_value = [&script](StringView script_file) { + .accept_value = [&script_parts](StringView script_file) { auto maybe_file = Core::File::open(script_file, Core::File::OpenMode::Read); if (maybe_file.is_error()) { warnln("Failed to open script file: {}", maybe_file.release_error()); @@ -1022,7 +1028,8 @@ ErrorOr serenity_main(Main::Arguments args) warnln("Failed to read contents of script file {}: {}", script_file, maybe_file_contents.release_error()); return false; } - return script.add_script_part(StringView { maybe_file_contents.release_value().bytes() }); + script_parts.append(StringView { maybe_file_contents.release_value().bytes() }); + return true; }, }); arg_parser.add_option(Core::ArgsParser::Option { @@ -1030,14 +1037,24 @@ ErrorOr serenity_main(Main::Arguments args) .help_string = "A script of commands", .short_name = 'e', .value_name = "script", - .accept_value = [&script](StringView script_argument) { - return script.add_script_part(script_argument); + .accept_value = [&script_parts](StringView script_argument) { + script_parts.append(script_argument); + return true; }, }); + arg_parser.add_ignored("regexp-extended", 'E'); + arg_parser.add_option(strict_posix, "Use POSIX regular expressions", "posix", 'p'); arg_parser.add_option(edit_in_place, "Edit file in place, implies -n", "in-place", 'i'); arg_parser.add_positional_argument(pos_args, "script and/or file", "...", Core::ArgsParser::Required::No); arg_parser.parse(args); + for (StringView part : script_parts) { + if (!script.add_script_part(part, strict_posix)) { + // Warning has already been emitted so just exit. + return 1; + } + } + // When editing in-place, there's also no default output. suppress_default_output |= edit_in_place; @@ -1050,7 +1067,7 @@ ErrorOr serenity_main(Main::Arguments args) warnln("No script specified, aborting"); return 1; } - if (!script.add_script_part(pos_args[0])) { + if (!script.add_script_part(pos_args[0], strict_posix)) { return 1; } pos_args.remove(0); diff --git a/Userland/Utilities/tar.cpp b/Userland/Utilities/tar.cpp index a42bc0a6f1622e..6a202ce33b4e10 100644 --- a/Userland/Utilities/tar.cpp +++ b/Userland/Utilities/tar.cpp @@ -238,7 +238,7 @@ ErrorOr serenity_main(Main::Arguments arguments) output_stream = TRY(Compress::LzmaCompressor::create_container(move(output_stream), {})); if (xz) - TODO(); + return Error::from_string_literal("Creating XZ compressed archives is not supported"); Archive::TarOutputStream tar_stream(move(output_stream)); diff --git a/Userland/Utilities/traceroute.cpp b/Userland/Utilities/traceroute.cpp index 52c4474eeb2186..fcc61a54cce5d9 100644 --- a/Userland/Utilities/traceroute.cpp +++ b/Userland/Utilities/traceroute.cpp @@ -6,10 +6,10 @@ #include #include +#include #include #include #include -#include #include #include #include @@ -85,7 +85,7 @@ ErrorOr serenity_main(Main::Arguments arguments) request.header = { ICMP_ECHO, 0, 0, { { 0, 0 } } }; bool fits = ttl_number.copy_characters_to_buffer(request.msg, sizeof(request.msg)); VERIFY(fits); - request.header.checksum = Crypto::Checksum::IPv4Header({ &request, sizeof(request) }).digest(); + request.header.checksum = bit_cast(InternetChecksum({ &request, sizeof(request) }).digest()); m_timer.start(); TRY(Core::System::sendto(fd, &request, sizeof(request), 0, (sockaddr*)&host_address, sizeof(host_address)));