diff --git a/.dockerignore b/.dockerignore index 86fab1489..30b551f9f 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,10 +1,8 @@ .github/* exhaust/* -pics/* remote-raw/* scripts/* .git .dockerignore README.md -LICENSE -Makefile \ No newline at end of file +LICENSE \ No newline at end of file diff --git a/.github/workflows/CI.yaml b/.github/workflows/CI.yaml index 56e58329f..b22062295 100644 --- a/.github/workflows/CI.yaml +++ b/.github/workflows/CI.yaml @@ -1,64 +1,43 @@ name: CI check on every PR on: pull_request: - branches: - - master paths-ignore: - '**.md' - 'Makefile' - 'config.json' jobs: - ci: - name: CI check on every push and PR - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@v3 - - - name: Install Go - uses: actions/setup-go@v3 - with: - go-version: '1.20' - - - name: Setup necessary packages - run: | - sudo apt install libvips-dev -y - - - name: run test cases - run: make && make test - image-test: - name: Check for image build and CVE + name: Check for image build, testing and CVE runs-on: ubuntu-latest + permissions: + pull-requests: write steps: - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: submodules: true - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 + uses: docker/setup-buildx-action@v3 - - name: Cache Docker layers - uses: actions/cache@v2 + - name: Make test + uses: docker/build-push-action@v6 with: - path: /tmp/.buildx-cache - key: ${{ runner.os }}-buildx-${{ github.sha }} - restore-keys: | - ${{ runner.os }}-buildx- + context: . + file: ./Dockerfile.CI + load: true + tags: | + ghcr.io/${{ github.event.repository.full_name }}:latest - - name: Build and load image - uses: docker/build-push-action@v3 + - name: Build image test + uses: docker/build-push-action@v6 with: context: . load: true tags: | ghcr.io/${{ github.event.repository.full_name }}:latest - cache-from: type=local,src=/tmp/.buildx-cache - cache-to: type=local,dest=/tmp/.buildx-cache-new,mode=max - - name: Install trivy run: | wget -qO - https://aquasecurity.github.io/trivy-repo/deb/public.key | sudo apt-key add - @@ -77,11 +56,12 @@ jobs: run: | echo "${{ steps.trivy.outputs.stdout }}" - - name: Comment PR + - name: Comment PR for CVE uses: thollander/actions-comment-pull-request@v2 with: message: | ``` ${{ steps.trivy.outputs.stdout }} ``` + comment_tag: cve GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/codecov.yaml b/.github/workflows/codecov.yaml index be556f609..b30c1a4ee 100644 --- a/.github/workflows/codecov.yaml +++ b/.github/workflows/codecov.yaml @@ -14,16 +14,16 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Install Go - uses: actions/setup-go@v3 + uses: actions/setup-go@v5 with: - go-version: '1.20' + go-version: '1.23' - name: Setup necessary packages run: | - sudo apt install libvips-dev -y + sudo apt update && sudo apt install libvips-dev -y - name: run test cases run: make && make test diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index edbca9b2e..2062add3a 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -15,7 +15,6 @@ on: push: branches: [ master ] pull_request: - # The branches below must be a subset of the branches above branches: [ master ] schedule: - cron: '32 20 * * 2' @@ -33,38 +32,23 @@ jobs: fail-fast: false matrix: language: [ 'go' ] - # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python', 'ruby' ] - # Learn more about CodeQL language support at https://git.io/codeql-language-support steps: - name: Checkout repository - uses: actions/checkout@v3 + uses: actions/checkout@v4 + + - name: Install Go + uses: actions/setup-go@v5 + with: + go-version: '1.23' - # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL uses: github/codeql-action/init@v2 with: languages: ${{ matrix.language }} - # If you wish to specify custom queries, you can do so here or in a config file. - # By default, queries listed here will override any specified in a config file. - # Prefix the list here with "+" to use these queries and those in the config file. - # queries: ./path/to/local/query, your-org/your-repo/queries@main - # Autobuild attempts to build any compiled languages (C/C++, C#, or Java). - # If this step fails, then you should remove it and run the build manually (see below) - name: Autobuild uses: github/codeql-action/autobuild@v2 - # ℹ️ Command-line programs to run using the OS shell. - # πŸ“š https://git.io/JvXDl - - # ✏️ If the Autobuild fails above, remove it and uncomment the following three lines - # and modify them (or add more) to build your code if your project - # uses a compiled language - - #- run: | - # make bootstrap - # make release - - name: Perform CodeQL Analysis uses: github/codeql-action/analyze@v2 diff --git a/.github/workflows/integration-test.yaml b/.github/workflows/integration-test.yaml index f2010178c..62dab8b42 100644 --- a/.github/workflows/integration-test.yaml +++ b/.github/workflows/integration-test.yaml @@ -14,10 +14,15 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout code - uses: actions/checkout@v3 + uses: actions/checkout@v4 + + - name: Install docker-compose + run: | + sudo apt-get update + sudo apt-get install -y docker-compose - name: Build image - uses: docker/build-push-action@v3 + uses: docker/build-push-action@v6 with: context: . platforms: linux/amd64 @@ -32,7 +37,7 @@ jobs: - name: Send Requests to Server run: | cd pics - find * -type f -print | xargs -I {} curl -H "User-Agent: Mozilla/5.0 Gecko/20100101 Firefox/98.0" http://localhost:3333/{} + find * -type f -print | xargs -I {} curl -o /dev/null -H "Accept: image/webp" --silent http://localhost:3333/{} - name: Get container RAM stats run: | @@ -43,10 +48,15 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout code - uses: actions/checkout@v3 - + uses: actions/checkout@v4 + + - name: Install docker-compose + run: | + sudo apt-get update + sudo apt-get install -y docker-compose + - name: Build image - uses: docker/build-push-action@v3 + uses: docker/build-push-action@v6 with: context: . platforms: linux/amd64 @@ -61,7 +71,7 @@ jobs: - name: Send Requests to Server run: | cd pics - find * -type f -print | xargs -I {} curl -H "User-Agent: Mozilla/5.0 Gecko/20100101 Firefox/98.0" http://localhost:3333/{} + find * -type f -print | xargs -I {} curl -o /dev/null -H "Accept: image/webp" --silent http://localhost:3333/{} - name: Get container RAM stats run: | diff --git a/.github/workflows/release_binary.yaml b/.github/workflows/release_binary.yaml index 16c5d6bac..81425c054 100644 --- a/.github/workflows/release_binary.yaml +++ b/.github/workflows/release_binary.yaml @@ -15,13 +15,13 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout code - uses: actions/checkout@v2 + uses: actions/checkout@v4 - name: Set up QEMU - uses: docker/setup-qemu-action@v1 + uses: docker/setup-qemu-action@v3 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 + uses: docker/setup-buildx-action@v3 - name: Make WebP Server Go (amd64) run: | diff --git a/.github/workflows/release_docker_image.yaml b/.github/workflows/release_docker_image.yaml index e31207fec..79b7ad4fc 100644 --- a/.github/workflows/release_docker_image.yaml +++ b/.github/workflows/release_docker_image.yaml @@ -15,7 +15,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: submodules: true @@ -43,10 +43,10 @@ jobs: password: ${{ secrets.GITHUB_TOKEN }} - name: Set up QEMU - uses: docker/setup-qemu-action@v1 + uses: docker/setup-qemu-action@v3 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v1 + uses: docker/setup-buildx-action@v3 - name: Cache Docker layers uses: actions/cache@v2 @@ -57,10 +57,10 @@ jobs: ${{ runner.os }}-buildx- - name: Build and push latest images - uses: docker/build-push-action@v3 + uses: docker/build-push-action@v6 with: context: . - platforms: linux/arm,linux/amd64,linux/arm64 + platforms: linux/amd64,linux/arm64 push: true tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} diff --git a/Dockerfile b/Dockerfile index 052a98a16..b09939960 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM golang:1.20-bookworm as builder +FROM golang:1.23-bookworm AS builder ARG IMG_PATH=/opt/pics ARG EXHAUST_PATH=/opt/exhaust @@ -14,7 +14,12 @@ RUN cd /build && sed -i "s|.\/pics|${IMG_PATH}|g" config.json \ FROM debian:bookworm-slim -RUN apt update && apt install --no-install-recommends libvips ca-certificates libjemalloc2 libtcmalloc-minimal4 -y && rm -rf /var/lib/apt/lists/* && rm -rf /var/cache/apt/archives/* +RUN apt update && apt install --no-install-recommends libvips ca-certificates libjemalloc2 libtcmalloc-minimal4 curl -y && rm -rf /var/lib/apt/lists/* && rm -rf /var/cache/apt/archives/* + +COPY ./assets /build/assets +# Install libam with correct arch +RUN dpkg -i /build/assets/libaom3_3.11.0-1_$(dpkg --print-architecture).deb && \ + rm /build/assets/libaom3_3.11.0-1_$(dpkg --print-architecture).deb COPY --from=builder /build/webp-server /usr/bin/webp-server COPY --from=builder /build/config.json /etc/config.json diff --git a/Dockerfile.CI b/Dockerfile.CI new file mode 100644 index 000000000..5983ec538 --- /dev/null +++ b/Dockerfile.CI @@ -0,0 +1,11 @@ +FROM golang:1.23-bookworm AS builder + +ARG IMG_PATH=/opt/pics +ARG EXHAUST_PATH=/opt/exhaust +RUN apt update && apt install --no-install-recommends libvips-dev -y && mkdir /build +COPY go.mod /build +RUN cd /build && go mod download + +COPY . /build + +RUN cd /build && make && make test \ No newline at end of file diff --git a/Makefile b/Makefile index 2c91d6232..9b11f5031 100644 --- a/Makefile +++ b/Makefile @@ -5,9 +5,11 @@ else endif ifeq ($(shell uname -m),aarch64) - ARCH=arm64 + ARCH=arm64 +else ifeq ($(shell uname -m),arm64) + ARCH=arm64 else - ARCH=amd64 + ARCH=amd64 endif default: @@ -24,7 +26,7 @@ tools-dir: install-staticcheck: tools-dir GOBIN=`pwd`/tools/bin go install honnef.co/go/tools/cmd/staticcheck@latest - curl -sfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh| sh -s -- -b ./tools/bin v1.52.2 + curl -sfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh| sh -s -- -b ./tools/bin v1.59.1 static-check: install-staticcheck #S1000,SA1015,SA4006,SA4011,S1023,S1034,ST1003,ST1005,ST1016,ST1020,ST1021 @@ -35,8 +37,7 @@ test: go test -v -coverprofile=coverage.txt -covermode=atomic ./... clean: - rm -rf prefetch remote-raw exhaust tools coverage.txt - + rm -rf prefetch remote-raw exhaust tools coverage.txt metadata exhaust_test docker: DOCKER_BUILDKIT=1 docker build -t webpsh/webps . diff --git a/README.md b/README.md index a8b532116..86d227532 100644 --- a/README.md +++ b/README.md @@ -8,17 +8,15 @@ [![codecov](https://codecov.io/gh/webp-sh/webp_server_go/branch/master/graph/badge.svg?token=VR3BMZME65)](https://codecov.io/gh/webp-sh/webp_server_go) ![Docker Pulls](https://img.shields.io/docker/pulls/webpsh/webp-server-go?style=plastic) -[Documentation](https://docs.webp.sh/) | [Website](https://webp.sh/) +[Documentation](https://docs.webp.sh/) | [Website](https://webp.sh/) | [Blog](https://blog.webp.se/) This is a Server based on Golang, which allows you to serve WebP images on the fly. -* currently supported image format: JPEG, PNG, BMP, GIF, SVG +Currently supported image format: JPEG, PNG, BMP, GIF, SVG, HEIC, NEF, WEBP -> e.g When you visit `https://your.website/pics/tsuki.jpg`，it will serve as `image/webp` format without changing the -> URL. +> e.g When you visit `https://your.website/pics/tsuki.jpg`，it will serve as `image/webp`/`image/avif` format without changing the URL. > -> ~~For Safari and Opera users, the original image will be used.~~ -> We've now supported Safari/Chrome/Firefox on iOS 14/iPadOS 14 +> GIF image will not be converted to AVIF format because the converted AVIF image is not animated. ## Usage with Docker(recommended) @@ -34,11 +32,10 @@ services: image: webpsh/webp-server-go # image: ghcr.io/webp-sh/webp_server_go restart: always - environment: - - MALLOC_ARENA_MAX=1 volumes: - ./path/to/pics:/opt/pics - ./exhaust:/opt/exhaust + - ./metadata:/opt/metadata ports: - 127.0.0.1:3333:3333 ``` @@ -52,7 +49,8 @@ Suppose your website and image has the following pattern. Then * `./path/to/pics` should be changed to `/var/www/img.webp.sh` -* `./exhaust` is cache folder for output images, by default it will be in `exhaust` directory alongside with `docker-compose.yml` file, if you'd like to keep cached images in another folder as , you can change `./exhaust` to `/some/other/path/to/exhaust` +* `./exhaust` is cache folder for output images, by default it will be in `exhaust` directory alongside with `docker-compose.yml` file, if you'd like to keep cached images in another folder, you can change `./exhaust` to `/some/other/path/to/exhaust` +* `./metadata` is cache folder for images' metadata, by default it will be in `metadata` directory alongside with `docker-compose.yml` file Start the container using: @@ -62,26 +60,58 @@ docker-compose up -d Now the server should be running on `127.0.0.1:3333`, visiting `http://127.0.0.1:3333/path/tsuki.jpg` will see the optimized version of `/var/www/img.webp.sh/path/tsuki.jpg`, you can now add reverse proxy to make it public, for example, let Nginx to `proxy_pass http://127.0.0.1:3333/;`, and your WebP Server is on-the-fly! -You can refer to [Docker | WebP Server Documentation](https://docs.webp.sh/usage/docker/) for more info, such as custom config, AVIF support etc. +## Custom config + +If you'd like to use a customized `config.json`, you can follow the steps in [Configuration | WebP Server Documentation](https://docs.webp.sh/usage/configuration/) to genereate one, and mount it into the container's `/etc/config.json`, example `docker-compose.yml` as follows: + +```yml +version: '3' + +services: + webp: + image: webpsh/webp-server-go + # image: ghcr.io/webp-sh/webp_server_go + restart: always + volumes: + - ./path/to/pics:/opt/pics + - ./path/to/exhaust:/opt/exhaust + - ./path/to/metadata:/opt/metadata + - ./config.json:/etc/config.json + ports: + - 127.0.0.1:3333:3333 +``` + +You can refer to [Configuration | WebP Server Documentation](https://docs.webp.sh/usage/configuration/) for more info, such as custom config, AVIF support etc. ## Advanced Usage -If you'd like to use with binary, please consult to [Basic Usage | WebP Server Documentation](https://docs.webp.sh/usage/basic-usage/), spoiler alert: you may encounter issues with `glibc` and some dependency libraries. +If you'd like to use with binary, please consult to [Use with Binary(Advanced) | WebP Server Documentation](https://docs.webp.sh/usage/usage-with-binary/) + +> spoiler alert: you may encounter issues with `glibc` and some dependency libraries. -For supervisor or detailed Nginx configuration, please read our documentation at [https://docs.webp.sh/](https://docs.webp.sh/) +For `supervisor` or detailed Nginx configuration, please read our documentation at [https://docs.webp.sh/](https://docs.webp.sh/) ## WebP Cloud Services -We are currently building a new service called [WebP Cloud Services](https://webp.se/), it now has two services: +We are currently building a new service called [WebP Cloud Services](https://webp.se/), it now has three parts: * [Public Service](https://public.webp.se) * GitHub Avatar/Gravater reverse proxy with WebP optimization, for example, change `https://www.gravatar.com/avatar/09eba3a443a7ea91cf818f6b27607d66` to `https://gravatar.webp.se/avatar/09eba3a443a7ea91cf818f6b27607d66` for rendering will get a smaller version of gravater, making your website faster - * Totally free service and currently has a large number of users, this includes, but is not limited to [CNX Software](https://medium.com/amarao/scaleway-arm-servers-50f85c4cefbe),[Indienova](https://indienova.com/en) + * Totally free service and currently has a large number of users, this includes, but is not limited to [CNX Software](https://www.cnx-software.com/), [Indienova](https://indienova.com/en) * [WebP Cloud](https://docs.webp.se/webp-cloud/) - * No need to install WebP Server Go, especially suitable for static websites. - * Image Conversion: WebP Cloud converts images to WebP format, reducing size while maintaining quality for faster website loading. - * Example: Original image URL (https://yyets.dmesg.app/api/user/avatar/Benny) becomes compressed URL (https://vz4w427.webp.ee/api/user/avatar/Benny). + * No need to install WebP Server Go yourself, especially suitable for static websites. + * Image Conversion: WebP Cloud converts images to WebP/AVIF format, reducing size while maintaining quality for faster website loading. + * Example 1: Original image URL (https://yyets.dmesg.app/api/user/avatar/BennyThink) becomes compressed URL (https://vz4w427.webp.ee/api/user/avatar/Benny). + * Example 2: Original image URL (https://yyets.dmesg.app/api/user/avatar/BennyThink) becomes a thumbnail image using URL (https://vz4w427.webp.ee/api/user/avatar/BennyThink?width=200). * Caching: WebP Cloud automatically caches served images, reducing traffic and bandwidth load on the origin server. +* [Fly](https://webp.se/fly/) + * We call this service Fly, with the aim of providing a public and free service that users can experience without registering on WebP Cloud. + As this is a public service, some limitations compared to WebP Cloud are imposed: + + - Fly supports a maximum original image size of 8MB, while WebP Cloud supports up to 80MB. + - Fly cache time is 1 day, while WebP Cloud has unlimited time (can be manually cleared at any time). + - It does not support parameters like `blur`, `sharpen` for image processing. + - And that’s it. For detailed information, please visit [WebP Cloud Services Website](https://webp.se/) or [WebP Cloud Services Docs](https://docs.webp.se/). diff --git a/assets/libaom3_3.11.0-1_amd64.deb b/assets/libaom3_3.11.0-1_amd64.deb new file mode 100644 index 000000000..6f05ed667 Binary files /dev/null and b/assets/libaom3_3.11.0-1_amd64.deb differ diff --git a/assets/libaom3_3.11.0-1_arm64.deb b/assets/libaom3_3.11.0-1_arm64.deb new file mode 100644 index 000000000..0ee1f4f67 Binary files /dev/null and b/assets/libaom3_3.11.0-1_arm64.deb differ diff --git a/config.json b/config.json index a6f76e10d..bb3e051b5 100644 --- a/config.json +++ b/config.json @@ -4,7 +4,13 @@ "QUALITY": "80", "IMG_PATH": "./pics", "EXHAUST_PATH": "./exhaust", - "ALLOWED_TYPES": ["jpg","png","jpeg","bmp","gif","svg"], - "ENABLE_AVIF": false, - "ENABLE_EXTRA_PARAMS": false -} + "IMG_MAP": {}, + "ALLOWED_TYPES": ["jpg","png","jpeg","gif","bmp","svg","heic","nef"], + "CONVERT_TYPES": ["webp"], + "STRIP_METADATA": true, + "ENABLE_EXTRA_PARAMS": false, + "READ_BUFFER_SIZE": 4096, + "CONCURRENCY": 262144, + "DISABLE_KEEPALIVE": false, + "CACHE_TTL": 259200 +} \ No newline at end of file diff --git a/config/config.go b/config/config.go index d225b46d5..ae1235b05 100644 --- a/config/config.go +++ b/config/config.go @@ -6,6 +6,9 @@ import ( "os" "regexp" "runtime" + "slices" + "strconv" + "strings" "time" "github.com/patrickmn/go-cache" @@ -17,76 +20,138 @@ const ( FiberLogFormat = "${ip} - [${time}] ${method} ${url} ${status} ${referer} ${ua}\n" WebpMax = 16383 AvifMax = 65536 - RemoteRaw = "remote-raw" - - SampleConfig = ` + HttpRegexp = `^https?://` + SampleConfig = ` { "HOST": "127.0.0.1", "PORT": "3333", "QUALITY": "80", "IMG_PATH": "./pics", "EXHAUST_PATH": "./exhaust", - "ALLOWED_TYPES": ["jpg","png","jpeg","bmp","svg"], - "ENABLE_AVIF": false, - "ENABLE_EXTRA_PARAMS": false + "IMG_MAP": {}, + "ALLOWED_TYPES": ["jpg","png","jpeg","gif","bmp","svg","heic","nef"], + "CONVERT_TYPES": ["webp"], + "STRIP_METADATA": true, + "ENABLE_EXTRA_PARAMS": false, + "EXTRA_PARAMS_CROP_INTERESTING": "InterestingAttention", + "READ_BUFFER_SIZE": 4096, + "CONCURRENCY": 262144, + "DISABLE_KEEPALIVE": false, + "CACHE_TTL": 259200, + "MAX_CACHE_SIZE": 0 }` - - SampleSystemd = ` -[Unit] -Description=WebP Server Go -Documentation=https://github.com/webp-sh/webp_server_go -After=nginx.target - -[Service] -Type=simple -StandardError=journal -WorkingDirectory=/opt/webps -ExecStart=/opt/webps/webp-server --config /opt/webps/config.json -Restart=always -RestartSec=3s - -[Install] -WantedBy=multi-user.target` ) var ( - ConfigPath string - Jobs int - DumpSystemd bool - DumpConfig bool - ShowVersion bool - ProxyMode bool - Prefetch bool - Config jsonFile - Version = "0.9.8" - WriteLock = cache.New(5*time.Minute, 10*time.Minute) + ConfigPath string + Jobs int + Verbosity int + DumpSystemd bool + DumpConfig bool + ShowVersion bool + ProxyMode bool + Prefetch bool // Prefech in go-routine, with WebP Server Go launch normally + PrefetchForeground bool // Standalone prefetch, prefetch and exit + AllowNonImage bool + Config = NewWebPConfig() + Version = "0.13.1" + WriteLock = cache.New(5*time.Minute, 10*time.Minute) + ConvertLock = cache.New(5*time.Minute, 10*time.Minute) + LocalHostAlias = "local" + RemoteCache *cache.Cache + DefaultAllowedTypes = []string{"jpg", "png", "jpeg", "bmp", "gif", "svg", "nef", "heic", "webp", "avif", "jxl"} // Default allowed image types ) -const Metadata = "metadata" +type ImageMeta struct { + Width int `json:"width"` + Height int `json:"height"` + Format string `json:"format"` + Size int `json:"size"` + NumPages int `json:"num_pages"` + Blurhash string `json:"blurhash"` + Colorspace string `json:"colorspace"` +} type MetaFile struct { Id string `json:"id"` // hash of below path️, also json file name id.webp Path string `json:"path"` // local: path with width and height, proxy: full url Checksum string `json:"checksum"` // hash of original file or hash(etag). Use this to identify changes + + ImageMeta } -type jsonFile struct { - Host string `json:"HOST"` - Port string `json:"PORT"` - ImgPath string `json:"IMG_PATH"` - Quality int `json:"QUALITY,string"` - AllowedTypes []string `json:"ALLOWED_TYPES"` - ExhaustPath string `json:"EXHAUST_PATH"` - EnableAVIF bool `json:"ENABLE_AVIF"` - EnableExtraParams bool `json:"ENABLE_EXTRA_PARAMS"` +type WebpConfig struct { + Host string `json:"HOST"` + Port string `json:"PORT"` + ImgPath string `json:"IMG_PATH"` + Quality int `json:"QUALITY,string"` + AllowedTypes []string `json:"ALLOWED_TYPES"` + ConvertTypes []string `json:"CONVERT_TYPES"` + ImageMap map[string]string `json:"IMG_MAP"` + ExhaustPath string `json:"EXHAUST_PATH"` + MetadataPath string `json:"METADATA_PATH"` + RemoteRawPath string `json:"REMOTE_RAW_PATH"` + + EnableWebP bool `json:"ENABLE_WEBP"` + EnableAVIF bool `json:"ENABLE_AVIF"` + EnableJXL bool `json:"ENABLE_JXL"` + + EnableExtraParams bool `json:"ENABLE_EXTRA_PARAMS"` + ExtraParamsCropInteresting string `json:"EXTRA_PARAMS_CROP_INTERESTING"` + + StripMetadata bool `json:"STRIP_METADATA"` + ReadBufferSize int `json:"READ_BUFFER_SIZE"` + Concurrency int `json:"CONCURRENCY"` + DisableKeepalive bool `json:"DISABLE_KEEPALIVE"` + CacheTTL int `json:"CACHE_TTL"` // In minutes + + MaxCacheSize int `json:"MAX_CACHE_SIZE"` // In MB, for max cached exhausted/metadata files(plus remote-raw if applicable), 0 means no limit +} + +func NewWebPConfig() *WebpConfig { + // Copy DefaultAllowedTypes to avoid modification + defaultAllowedTypes := make([]string, len(DefaultAllowedTypes)) + copy(defaultAllowedTypes, DefaultAllowedTypes) + return &WebpConfig{ + Host: "0.0.0.0", + Port: "3333", + ImgPath: "./pics", + Quality: 80, + AllowedTypes: defaultAllowedTypes, + ConvertTypes: []string{"webp"}, + ImageMap: map[string]string{}, + ExhaustPath: "./exhaust", + MetadataPath: "./metadata", + RemoteRawPath: "./remote-raw", + + EnableWebP: false, + EnableAVIF: false, + EnableJXL: false, + + EnableExtraParams: false, + ExtraParamsCropInteresting: "InterestingAttention", + StripMetadata: true, + ReadBufferSize: 4096, + Concurrency: 262144, + DisableKeepalive: false, + CacheTTL: 259200, + + MaxCacheSize: 0, + } } func init() { flag.StringVar(&ConfigPath, "config", "config.json", "/path/to/config.json. (Default: ./config.json)") - flag.BoolVar(&Prefetch, "prefetch", false, "Prefetch and convert image to webp") + flag.BoolVar(&Prefetch, "prefetch", false, "Prefetch and convert images to optimized format, with WebP Server Go launch normally") + flag.BoolVar(&PrefetchForeground, "prefetch-foreground", false, "Prefetch and convert image to optimized format in foreground, prefetch and exit") flag.IntVar(&Jobs, "jobs", runtime.NumCPU(), "Prefetch thread, default is all.") - flag.BoolVar(&DumpConfig, "dump-config", false, "Print sample config.json") - flag.BoolVar(&DumpSystemd, "dump-systemd", false, "Print sample systemd service file.") + // 0 = silent (no log messages) + // 1 = error (error messages only) + // 2 = warn (error messages and warnings only) + // 3 = info (error messages, warnings and normal activity logs) + // 4 = debug (all info plus additional messages for debugging) + flag.IntVar(&Verbosity, "verbosity", 3, "Log level(0: silent, 1: error, 2: warn, 3:info, 4: debug), default to 3: info") + flag.BoolVar(&DumpConfig, "dump-config", false, "Print sample config.json.") flag.BoolVar(&ShowVersion, "V", false, "Show version information.") } @@ -99,16 +164,173 @@ func LoadConfig() { _ = decoder.Decode(&Config) _ = jsonObject.Close() switchProxyMode() + Config.ImageMap = parseImgMap(Config.ImageMap) + + if slices.Contains(Config.ConvertTypes, "webp") { + Config.EnableWebP = true + } + if slices.Contains(Config.ConvertTypes, "avif") { + Config.EnableAVIF = true + } + if slices.Contains(Config.ConvertTypes, "jxl") { + Config.EnableJXL = true + } + + // Read from ENV for override + if os.Getenv("WEBP_HOST") != "" { + Config.Host = os.Getenv("WEBP_HOST") + } + if os.Getenv("WEBP_PORT") != "" { + Config.Port = os.Getenv("WEBP_PORT") + } + if os.Getenv("WEBP_IMG_PATH") != "" { + Config.ImgPath = os.Getenv("WEBP_IMG_PATH") + } + if os.Getenv("WEBP_EXHAUST_PATH") != "" { + Config.ExhaustPath = os.Getenv("WEBP_EXHAUST_PATH") + } + if os.Getenv("WEBP_QUALITY") != "" { + quality, err := strconv.Atoi(os.Getenv("WEBP_QUALITY")) + if err != nil { + log.Warnf("WEBP_QUALITY is not a valid integer, using value in config.json %d", Config.Quality) + } else { + Config.Quality = quality + } + } + if os.Getenv("WEBP_ALLOWED_TYPES") != "" { + Config.AllowedTypes = strings.Split(os.Getenv("WEBP_ALLOWED_TYPES"), ",") + } + + // Override enabled convert types + if os.Getenv("WEBP_CONVERT_TYPES") != "" { + Config.ConvertTypes = strings.Split(os.Getenv("WEBP_CONVERT_TYPES"), ",") + Config.EnableWebP = false + Config.EnableAVIF = false + Config.EnableJXL = false + if slices.Contains(Config.ConvertTypes, "webp") { + Config.EnableWebP = true + } + if slices.Contains(Config.ConvertTypes, "avif") { + Config.EnableAVIF = true + } + if slices.Contains(Config.ConvertTypes, "jxl") { + Config.EnableJXL = true + } + } + + if os.Getenv("WEBP_ENABLE_EXTRA_PARAMS") != "" { + enableExtraParams := os.Getenv("WEBP_ENABLE_EXTRA_PARAMS") + if enableExtraParams == "true" { + Config.EnableExtraParams = true + } else if enableExtraParams == "false" { + Config.EnableExtraParams = false + } else { + log.Warnf("WEBP_ENABLE_EXTRA_PARAMS is not a valid boolean, using value in config.json %t", Config.EnableExtraParams) + } + } + if os.Getenv("WEBP_EXTRA_PARAMS_CROP_INTERESTING") != "" { + availableInteresting := []string{"InterestingNone", "InterestingEntropy", "InterestingCentre", "InterestingAttention", "InterestringLow", "InterestingHigh", "InterestingAll"} + if slices.Contains(availableInteresting, os.Getenv("WEBP_EXTRA_PARAMS_CROP_INTERESTING")) { + Config.ExtraParamsCropInteresting = os.Getenv("WEBP_EXTRA_PARAMS_CROP_INTERESTING") + } else { + log.Warnf("WEBP_EXTRA_PARAMS_CROP_INTERESTING is not a valid interesting, using value in config.json %s", Config.ExtraParamsCropInteresting) + } + } + + if os.Getenv("WEBP_STRIP_METADATA") != "" { + stripMetadata := os.Getenv("WEBP_STRIP_METADATA") + if stripMetadata == "true" { + Config.StripMetadata = true + } else if stripMetadata == "false" { + Config.StripMetadata = false + } else { + log.Warnf("WEBP_STRIP_METADATA is not a valid boolean, using value in config.json %t", Config.StripMetadata) + } + } + if os.Getenv("WEBP_IMG_MAP") != "" { + // TODO + } + if os.Getenv("WEBP_READ_BUFFER_SIZE") != "" { + readBufferSize, err := strconv.Atoi(os.Getenv("WEBP_READ_BUFFER_SIZE")) + if err != nil { + log.Warnf("WEBP_READ_BUFFER_SIZE is not a valid integer, using value in config.json %d", Config.ReadBufferSize) + } else { + Config.ReadBufferSize = readBufferSize + } + } + if os.Getenv("WEBP_CONCURRENCY") != "" { + concurrency, err := strconv.Atoi(os.Getenv("WEBP_CONCURRENCY")) + if err != nil { + log.Warnf("WEBP_CONCURRENCY is not a valid integer, using value in config.json %d", Config.Concurrency) + } else { + Config.Concurrency = concurrency + } + } + if os.Getenv("WEBP_DISABLE_KEEPALIVE") != "" { + disableKeepalive := os.Getenv("WEBP_DISABLE_KEEPALIVE") + if disableKeepalive == "true" { + Config.DisableKeepalive = true + } else if disableKeepalive == "false" { + Config.DisableKeepalive = false + } else { + log.Warnf("WEBP_DISABLE_KEEPALIVE is not a valid boolean, using value in config.json %t", Config.DisableKeepalive) + } + } + if os.Getenv("WEBP_CACHE_TTL") != "" { + cacheTTL, err := strconv.Atoi(os.Getenv("WEBP_CACHE_TTL")) + if err != nil { + log.Warnf("WEBP_CACHE_TTL is not a valid integer, using value in config.json %d", Config.CacheTTL) + } else { + Config.CacheTTL = cacheTTL + } + } + + if Config.CacheTTL == 0 { + RemoteCache = cache.New(cache.NoExpiration, 10*time.Minute) + } else { + RemoteCache = cache.New(time.Duration(Config.CacheTTL)*time.Minute, 10*time.Minute) + } + + if os.Getenv("WEBP_MAX_CACHE_SIZE") != "" { + maxCacheSize, err := strconv.Atoi(os.Getenv("WEBP_MAX_CACHE_SIZE")) + if err != nil { + log.Warnf("WEBP_MAX_CACHE_SIZE is not a valid integer, using value in config.json %d", Config.MaxCacheSize) + } else { + Config.MaxCacheSize = maxCacheSize + } + } + + log.Debugln("Config init complete") + log.Debugln("Config", Config) +} + +func parseImgMap(imgMap map[string]string) map[string]string { + var parsedImgMap = map[string]string{} + httpRegexpMatcher := regexp.MustCompile(HttpRegexp) + for uriMap, uriMapTarget := range imgMap { + if httpRegexpMatcher.Match([]byte(uriMap)) || strings.HasPrefix(uriMap, "/") { + // Valid + parsedImgMap[uriMap] = uriMapTarget + } else { + // Invalid + log.Warnf("IMG_MAP key '%s' does matches '%s' or starts with '/' - skipped", uriMap, HttpRegexp) + } + } + return parsedImgMap } type ExtraParams struct { - Width int // in px - Height int // in px + Width int // in px + Height int // in px + MaxWidth int // in px + MaxHeight int // in px } func switchProxyMode() { - matched, _ := regexp.MatchString(`^https?://`, Config.ImgPath) + matched, _ := regexp.MatchString(HttpRegexp, Config.ImgPath) if matched { + // Enable proxy based on ImgPath should be deprecated in future versions + log.Warn("Enable proxy based on ImgPath will be deprecated in future versions. Use IMG_MAP config options instead") ProxyMode = true } } diff --git a/config/config_test.go b/config/config_test.go index dbb82eb32..9ef00160f 100644 --- a/config/config_test.go +++ b/config/config_test.go @@ -19,7 +19,10 @@ func TestLoadConfig(t *testing.T) { assert.Equal(t, Config.Port, "3333") assert.Equal(t, Config.Quality, 80) assert.Equal(t, Config.ImgPath, "./pics") + assert.Equal(t, Config.ImageMap, map[string]string{}) assert.Equal(t, Config.ExhaustPath, "./exhaust") + assert.Equal(t, Config.CacheTTL, 259200) + assert.Equal(t, Config.MaxCacheSize, 0) } func TestSwitchProxyMode(t *testing.T) { @@ -29,3 +32,26 @@ func TestSwitchProxyMode(t *testing.T) { switchProxyMode() assert.True(t, ProxyMode) } + +func TestParseImgMap(t *testing.T) { + empty := map[string]string{} + good := map[string]string{ + "/1": "../pics/dir1", + "http://example.com": "../pics", + "https://example.com": "../pics", + } + bad := map[string]string{ + "1": "../pics/dir1", + "httpx://example.com": "../pics", + "ftp://example.com": "../pics", + } + + assert.Equal(t, empty, parseImgMap(empty)) + assert.Equal(t, empty, parseImgMap(bad)) + assert.Equal(t, good, parseImgMap(good)) + + for k, v := range good { + bad[k] = v + } + assert.Equal(t, good, parseImgMap(bad)) +} diff --git a/encoder/encoder.go b/encoder/encoder.go index f7b8586f1..5b56e5acd 100644 --- a/encoder/encoder.go +++ b/encoder/encoder.go @@ -1,12 +1,12 @@ package encoder import ( - "errors" "os" "path" "runtime" "strings" "sync" + "time" "webp_server_go/config" "webp_server_go/helper" @@ -17,46 +17,54 @@ import ( var ( boolFalse vips.BoolParameter intMinusOne vips.IntParameter + // Source image encoder ignore list for WebP and AVIF + // We shouldn't convert Unknown and AVIF to WebP + webpIgnore = []vips.ImageType{vips.ImageTypeUnknown, vips.ImageTypeAVIF} + // We shouldn't convert Unknown,AVIF and GIF to AVIF + avifIgnore = append(webpIgnore, vips.ImageTypeGIF) ) func init() { + vips.LoggingSettings(nil, vips.LogLevelError) vips.Startup(&vips.Config{ ConcurrencyLevel: runtime.NumCPU(), }) boolFalse.Set(false) intMinusOne.Set(-1) +} +func loadImage(filename string) (*vips.ImageRef, error) { + img, err := vips.LoadImageFromFile(filename, &vips.ImportParams{ + FailOnError: boolFalse, + NumPages: intMinusOne, + }) + return img, err } -func resizeImage(img *vips.ImageRef, extraParams config.ExtraParams) error { - imgHeightWidthRatio := float32(img.Metadata().Height) / float32(img.Metadata().Width) - if extraParams.Width > 0 && extraParams.Height > 0 { - err := img.Thumbnail(extraParams.Width, extraParams.Height, vips.InterestingAttention) - if err != nil { - return err - } - } else if extraParams.Width > 0 && extraParams.Height == 0 { - err := img.Thumbnail(extraParams.Width, int(float32(extraParams.Width)*imgHeightWidthRatio), 0) - if err != nil { - return err - } - } else if extraParams.Height > 0 && extraParams.Width == 0 { - err := img.Thumbnail(int(float32(extraParams.Height)/imgHeightWidthRatio), extraParams.Height, 0) - if err != nil { - return err +func ConvertFilter(rawPath, jxlPath, avifPath, webpPath string, extraParams config.ExtraParams, supportedFormats map[string]bool, c chan int) { + // Wait for the conversion to complete and return the converted image + retryDelay := 100 * time.Millisecond // Initial retry delay + + for { + if _, found := config.ConvertLock.Get(rawPath); found { + log.Debugf("file %s is locked under conversion, retrying in %s", rawPath, retryDelay) + time.Sleep(retryDelay) + } else { + // The lock is released, indicating that the conversion is complete + break } } - return nil -} -func ConvertFilter(raw, avifPath, webpPath string, extraParams config.ExtraParams, c chan int) { - // all absolute paths + // If there is a lock here, it means that another thread is converting the same image + // Lock rawPath to prevent concurrent conversion + config.ConvertLock.Set(rawPath, true, -1) + defer config.ConvertLock.Delete(rawPath) var wg sync.WaitGroup - wg.Add(2) - if !helper.ImageExists(avifPath) && config.Config.EnableAVIF { + wg.Add(3) + if !helper.ImageExists(avifPath) && config.Config.EnableAVIF && supportedFormats["avif"] { go func() { - err := convertImage(raw, avifPath, "avif", extraParams) + err := convertImage(rawPath, avifPath, "avif", extraParams) if err != nil { log.Errorln(err) } @@ -66,9 +74,9 @@ func ConvertFilter(raw, avifPath, webpPath string, extraParams config.ExtraParam wg.Done() } - if !helper.ImageExists(webpPath) { + if !helper.ImageExists(webpPath) && config.Config.EnableWebP && supportedFormats["webp"] { go func() { - err := convertImage(raw, webpPath, "webp", extraParams) + err := convertImage(rawPath, webpPath, "webp", extraParams) if err != nil { log.Errorln(err) } @@ -77,6 +85,19 @@ func ConvertFilter(raw, avifPath, webpPath string, extraParams config.ExtraParam } else { wg.Done() } + + if !helper.ImageExists(jxlPath) && config.Config.EnableJXL && supportedFormats["jxl"] { + go func() { + err := convertImage(rawPath, jxlPath, "jxl", extraParams) + if err != nil { + log.Errorln(err) + } + defer wg.Done() + }() + } else { + wg.Done() + } + wg.Wait() if c != nil { @@ -84,90 +105,127 @@ func ConvertFilter(raw, avifPath, webpPath string, extraParams config.ExtraParam } } -func ResizeItself(raw, dest string, extraParams config.ExtraParams) { - log.Infof("Resize %s itself to %s", raw, dest) - img, _ := vips.LoadImageFromFile(raw, &vips.ImportParams{ - FailOnError: boolFalse, - }) - _ = resizeImage(img, extraParams) - buf, _, _ := img.ExportNative() - _ = os.WriteFile(dest, buf, 0600) - img.Close() -} - -func convertImage(raw, optimized, imageType string, extraParams config.ExtraParams) error { +func convertImage(rawPath, optimizedPath, imageType string, extraParams config.ExtraParams) error { // we need to create dir first - var err = os.MkdirAll(path.Dir(optimized), 0755) + var err = os.MkdirAll(path.Dir(optimizedPath), 0755) if err != nil { log.Error(err.Error()) } + // If original image is NEF, convert NEF image to JPG first + if strings.HasSuffix(strings.ToLower(rawPath), ".nef") { + var convertedRaw, converted = ConvertRawToJPG(rawPath, optimizedPath) + // If converted, use converted file as raw + if converted { + // Use converted file(JPG) as raw input for further conversion + rawPath = convertedRaw + // Remove converted file after conversion + defer func() { + log.Infoln("Removing intermediate conversion file:", convertedRaw) + err := os.Remove(convertedRaw) + if err != nil { + log.Warnln("failed to delete converted file", err) + } + }() + } + } + + // Image is only opened here + img, err := loadImage(rawPath) + defer img.Close() + + // Pre-process image(auto rotate, resize, etc.) + err = preProcessImage(img, imageType, extraParams) + if err != nil { + log.Warnf("Can't pre-process source image: %v", err) + } + + // If image is already in the target format, just copy it + imageFormat := img.Format() switch imageType { case "webp": - err = webpEncoder(raw, optimized, extraParams) + if imageFormat == vips.ImageTypeWEBP { + log.Infof("Image is already in WebP format, copying %s to %s", rawPath, optimizedPath) + return helper.CopyFile(rawPath, optimizedPath) + } else { + err = webpEncoder(img, rawPath, optimizedPath) + } case "avif": - err = avifEncoder(raw, optimized, extraParams) - } - return err -} - -func imageIgnore(imageFormat vips.ImageType) bool { - // Ignore Unknown, WebP, AVIF - ignoreList := []vips.ImageType{vips.ImageTypeUnknown, vips.ImageTypeWEBP, vips.ImageTypeAVIF} - for _, ignore := range ignoreList { - if imageFormat == ignore { - // Return err to render original image - return true + if imageFormat == vips.ImageTypeAVIF { + log.Infof("Image is already in AVIF format, copying %s to %s", rawPath, optimizedPath) + return helper.CopyFile(rawPath, optimizedPath) + } else { + err = avifEncoder(img, rawPath, optimizedPath) + } + case "jxl": + if imageFormat == vips.ImageTypeJXL { + log.Infof("Image is already in JXL format, copying %s to %s", rawPath, optimizedPath) + return helper.CopyFile(rawPath, optimizedPath) + } else { + err = jxlEncoder(img, rawPath, optimizedPath) } } - return false + + return err } -func avifEncoder(p1, p2 string, extraParams config.ExtraParams) error { - // if convert fails, return error; success nil +func jxlEncoder(img *vips.ImageRef, rawPath string, optimizedPath string) error { var ( buf []byte quality = config.Config.Quality + err error ) - img, err := vips.LoadImageFromFile(p1, &vips.ImportParams{ - FailOnError: boolFalse, - }) - if err != nil { - return err - } - - if imageIgnore(img.Format()) { - return errors.New("encoder: ignore image type") - } - if config.Config.EnableExtraParams { - err = resizeImage(img, extraParams) - if err != nil { - return err - } + // If quality >= 100, we use lossless mode + if quality >= 100 { + buf, _, err = img.ExportJxl(&vips.JxlExportParams{ + Effort: 1, + Tier: 4, + Lossless: true, + Distance: 1.0, + }) + } else { + buf, _, err = img.ExportJxl(&vips.JxlExportParams{ + Effort: 1, + Tier: 4, + Quality: quality, + Lossless: false, + Distance: 1.0, + }) } - // AVIF has a maximum resolution of 65536 x 65536 pixels. - if img.Metadata().Width > config.AvifMax || img.Metadata().Height > config.AvifMax { - return errors.New("AVIF: image too large") + if err != nil { + log.Warnf("Can't encode source image: %v to JXL", err) + return err } - err = img.AutoRotate() - if err != nil { + if err := os.WriteFile(optimizedPath, buf, 0600); err != nil { + log.Error(err) return err } + convertLog("JXL", rawPath, optimizedPath, quality) + return nil +} + +func avifEncoder(img *vips.ImageRef, rawPath string, optimizedPath string) error { + var ( + buf []byte + quality = config.Config.Quality + err error + ) + // If quality >= 100, we use lossless mode if quality >= 100 { buf, _, err = img.ExportAvif(&vips.AvifExportParams{ Lossless: true, - StripMetadata: true, + StripMetadata: config.Config.StripMetadata, }) } else { buf, _, err = img.ExportAvif(&vips.AvifExportParams{ Quality: quality, Lossless: false, - StripMetadata: true, + StripMetadata: config.Config.StripMetadata, }) } @@ -176,52 +234,22 @@ func avifEncoder(p1, p2 string, extraParams config.ExtraParams) error { return err } - if err := os.WriteFile(p2, buf, 0600); err != nil { + if err := os.WriteFile(optimizedPath, buf, 0600); err != nil { log.Error(err) return err } - img.Close() - convertLog("AVIF", p1, p2, quality) + convertLog("AVIF", rawPath, optimizedPath, quality) return nil } -func webpEncoder(p1, p2 string, extraParams config.ExtraParams) error { - // if convert fails, return error; success nil +func webpEncoder(img *vips.ImageRef, rawPath string, optimizedPath string) error { var ( buf []byte quality = config.Config.Quality + err error ) - img, err := vips.LoadImageFromFile(p1, &vips.ImportParams{ - FailOnError: boolFalse, - NumPages: intMinusOne, - }) - if err != nil { - return err - } - - if imageIgnore(img.Format()) { - return errors.New("encoder: ignore image type") - } - - if config.Config.EnableExtraParams { - err = resizeImage(img, extraParams) - if err != nil { - return err - } - } - - // The maximum pixel dimensions of a WebP image is 16383 x 16383. - if (img.Metadata().Width > config.WebpMax || img.Metadata().Height > config.WebpMax) && img.Format() != vips.ImageTypeGIF { - return errors.New("WebP: image too large") - } - - err = img.AutoRotate() - if err != nil { - return err - } - // If quality >= 100, we use lossless mode if quality >= 100 { // Lossless mode will not encounter problems as below, because in libvips as code below @@ -229,17 +257,17 @@ func webpEncoder(p1, p2 string, extraParams config.ExtraParams) error { // use_lossless_preset = 0; // disable -z option buf, _, err = img.ExportWebp(&vips.WebpExportParams{ Lossless: true, - StripMetadata: true, + StripMetadata: config.Config.StripMetadata, }) } else { - // If some special images cannot encode with default ReductionEffort(0), then try with 4 + // If some special images cannot encode with default ReductionEffort(0), then retry from 0 to 6 // Example: https://github.com/webp-sh/webp_server_go/issues/234 ep := vips.WebpExportParams{ Quality: quality, Lossless: false, - StripMetadata: true, + StripMetadata: config.Config.StripMetadata, } - for i := 0; i <= 6; i++ { + for i := range 7 { ep.ReductionEffort = i buf, _, err = img.ExportWebp(&ep) if err != nil && strings.Contains(err.Error(), "unable to encode") { @@ -251,7 +279,6 @@ func webpEncoder(p1, p2 string, extraParams config.ExtraParams) error { } } buf, _, err = img.ExportWebp(&ep) - } if err != nil { @@ -259,28 +286,27 @@ func webpEncoder(p1, p2 string, extraParams config.ExtraParams) error { return err } - if err := os.WriteFile(p2, buf, 0600); err != nil { + if err := os.WriteFile(optimizedPath, buf, 0600); err != nil { log.Error(err) return err } - img.Close() - convertLog("WebP", p1, p2, quality) + convertLog("WebP", rawPath, optimizedPath, quality) return nil } -func convertLog(itype, p1 string, p2 string, quality int) { - oldf, err := os.Stat(p1) +func convertLog(itype, rawPath string, optimizedPath string, quality int) { + oldf, err := os.Stat(rawPath) if err != nil { log.Error(err) return } - newf, err := os.Stat(p2) + newf, err := os.Stat(optimizedPath) if err != nil { log.Error(err) return } log.Infof("%s@%d%%: %s->%s %d->%d %.2f%% deflated", itype, quality, - p1, p2, oldf.Size(), newf.Size(), float32(newf.Size())/float32(oldf.Size())*100) + rawPath, optimizedPath, oldf.Size(), newf.Size(), float32(newf.Size())/float32(oldf.Size())*100) } diff --git a/encoder/prefetch.go b/encoder/prefetch.go index dfb4256c5..b222b9471 100644 --- a/encoder/prefetch.go +++ b/encoder/prefetch.go @@ -18,7 +18,7 @@ func PrefetchImages() { var sTime = time.Now() log.Infof("Prefetching using %d cores", config.Jobs) var finishChan = make(chan int, config.Jobs) - for i := 0; i < config.Jobs; i++ { + for range config.Jobs { finishChan <- 1 } @@ -33,12 +33,36 @@ func PrefetchImages() { if info.IsDir() { return nil } + // Only convert files with image extensions, use smaller of config.DefaultAllowedTypes and config.Config.AllowedTypes + if helper.CheckAllowedExtension(picAbsPath) { + // File type is allowed by user, check if it is an image + if helper.CheckImageExtension(picAbsPath) { + // File is an image, continue + } else { + return nil + } + } else { + return nil + } + // RawImagePath string, ImgFilename string, reqURI string - metadata := helper.ReadMetadata(picAbsPath, "") - avif, webp := helper.GenOptimizedAbsPath(metadata) - _ = os.MkdirAll(path.Dir(avif), 0755) + metadata := helper.ReadMetadata(picAbsPath, "", config.LocalHostAlias) + avifAbsPath, webpAbsPath, jxlAbsPath := helper.GenOptimizedAbsPath(metadata, config.LocalHostAlias) + + // Using avifAbsPath here is the same as using webpAbsPath/jxlAbsPath + _ = os.MkdirAll(path.Dir(avifAbsPath), 0755) + log.Infof("Prefetching %s", picAbsPath) - go ConvertFilter(picAbsPath, avif, webp, config.ExtraParams{Width: 0, Height: 0}, finishChan) + + // Allow all supported formats + supported := map[string]bool{ + "raw": true, + "webp": true, + "avif": true, + "jxl": true, + } + + go ConvertFilter(picAbsPath, jxlAbsPath, avifAbsPath, webpAbsPath, config.ExtraParams{Width: 0, Height: 0}, supported, finishChan) _ = bar.Add(<-finishChan) return nil }) diff --git a/encoder/process.go b/encoder/process.go new file mode 100644 index 000000000..6bb0fedf1 --- /dev/null +++ b/encoder/process.go @@ -0,0 +1,174 @@ +package encoder + +import ( + "errors" + "os" + "path" + "slices" + "webp_server_go/config" + + "github.com/davidbyttow/govips/v2/vips" + log "github.com/sirupsen/logrus" +) + +func resizeImage(img *vips.ImageRef, extraParams config.ExtraParams) error { + imageHeight := img.Height() + imageWidth := img.Width() + + imgHeightWidthRatio := float32(imageHeight) / float32(imageWidth) + + // Here we have width, height and max_width, max_height + // Both pairs cannot be used at the same time + + // max_height and max_width are used to make sure bigger images are resized to max_height and max_width + // e.g, 500x500px image with max_width=200,max_height=100 will be resized to 100x100 + // while smaller images are untouched + + // If both are used, we will use width and height + + if extraParams.MaxHeight > 0 && extraParams.MaxWidth > 0 { + // If any of it exceeds + if imageHeight > extraParams.MaxHeight || imageWidth > extraParams.MaxWidth { + // Check which dimension exceeds most + heightExceedRatio := float32(imageHeight) / float32(extraParams.MaxHeight) + widthExceedRatio := float32(imageWidth) / float32(extraParams.MaxWidth) + // If height exceeds more, like 500x500 -> 200x100 (2.5 < 5) + // Take max_height as new height ,resize and retain ratio + if heightExceedRatio > widthExceedRatio { + err := img.Thumbnail(int(float32(extraParams.MaxHeight)/imgHeightWidthRatio), extraParams.MaxHeight, 0) + if err != nil { + return err + } + } else { + err := img.Thumbnail(extraParams.MaxWidth, int(float32(extraParams.MaxWidth)*imgHeightWidthRatio), 0) + if err != nil { + return err + } + } + } + } + + if extraParams.MaxHeight > 0 && imageHeight > extraParams.MaxHeight && extraParams.MaxWidth == 0 { + err := img.Thumbnail(int(float32(extraParams.MaxHeight)/imgHeightWidthRatio), extraParams.MaxHeight, 0) + if err != nil { + return err + } + } + + if extraParams.MaxWidth > 0 && imageWidth > extraParams.MaxWidth && extraParams.MaxHeight == 0 { + err := img.Thumbnail(extraParams.MaxWidth, int(float32(extraParams.MaxWidth)*imgHeightWidthRatio), 0) + if err != nil { + return err + } + } + + if extraParams.Width > 0 && extraParams.Height > 0 { + var cropInteresting vips.Interesting + switch config.Config.ExtraParamsCropInteresting { + case "InterestingNone": + cropInteresting = vips.InterestingNone + case "InterestingCentre": + cropInteresting = vips.InterestingCentre + case "InterestingEntropy": + cropInteresting = vips.InterestingEntropy + case "InterestingAttention": + cropInteresting = vips.InterestingAttention + case "InterestingLow": + cropInteresting = vips.InterestingLow + case "InterestingHigh": + cropInteresting = vips.InterestingHigh + case "InterestingAll": + cropInteresting = vips.InterestingAll + default: + cropInteresting = vips.InterestingAttention + } + + err := img.Thumbnail(extraParams.Width, extraParams.Height, cropInteresting) + if err != nil { + return err + } + } + if extraParams.Width > 0 && extraParams.Height == 0 { + err := img.Thumbnail(extraParams.Width, int(float32(extraParams.Width)*imgHeightWidthRatio), 0) + if err != nil { + return err + } + } + if extraParams.Height > 0 && extraParams.Width == 0 { + err := img.Thumbnail(int(float32(extraParams.Height)/imgHeightWidthRatio), extraParams.Height, 0) + if err != nil { + return err + } + } + return nil +} + +func ResizeItself(raw, dest string, extraParams config.ExtraParams) { + log.Infof("Resize %s itself to %s", raw, dest) + + // we need to create dir first + var err = os.MkdirAll(path.Dir(dest), 0755) + if err != nil { + log.Error(err.Error()) + } + + img, err := vips.LoadImageFromFile(raw, &vips.ImportParams{ + FailOnError: boolFalse, + NumPages: intMinusOne, + }) + if err != nil { + log.Warnf("Could not load %s: %s", raw, err) + return + } + _ = resizeImage(img, extraParams) + if config.Config.StripMetadata { + img.RemoveMetadata() + } + buf, _, _ := img.ExportNative() + _ = os.WriteFile(dest, buf, 0600) + img.Close() +} + +// Pre-process image(auto rotate, resize, etc.) +func preProcessImage(img *vips.ImageRef, imageType string, extraParams config.ExtraParams) error { + // Check Width/Height and ignore image formats + switch imageType { + case "webp": + if img.Metadata().Width > config.WebpMax || img.Metadata().Height > config.WebpMax { + return errors.New("WebP: image too large") + } + imageFormat := img.Format() + if slices.Contains(webpIgnore, imageFormat) { + // Return err to render original image + return errors.New("WebP encoder: ignore image type") + } + case "avif": + if img.Metadata().Width > config.AvifMax || img.Metadata().Height > config.AvifMax { + return errors.New("AVIF: image too large") + } + imageFormat := img.Format() + if slices.Contains(avifIgnore, imageFormat) { + // Return err to render original image + return errors.New("AVIF encoder: ignore image type") + } + } + + if config.Config.EnableExtraParams { + err := resizeImage(img, extraParams) + if err != nil { + return err + } + } + // Skip auto rotate for GIF/WebP + if img.Format() == vips.ImageTypeGIF || img.Format() == vips.ImageTypeWEBP { + return nil + } else { + // Auto rotate + err := img.AutoRotate() + if err != nil { + return err + } + } + + return nil +} diff --git a/encoder/process_test.go b/encoder/process_test.go new file mode 100644 index 000000000..82ed925f3 --- /dev/null +++ b/encoder/process_test.go @@ -0,0 +1,90 @@ +package encoder + +import ( + "testing" + "webp_server_go/config" + + "github.com/davidbyttow/govips/v2/vips" +) + +func TestResizeImage(t *testing.T) { + img, _ := vips.Black(500, 500) + + // Define the parameters for the test cases + testCases := []struct { + extraParams config.ExtraParams // Extra parameters + expectedH int // Expected height + expectedW int // Expected width + }{ + // Tests for MaxHeight and MaxWidth + // Both extraParams.MaxHeight and extraParams.MaxWidth are 0 + { + extraParams: config.ExtraParams{ + MaxHeight: 0, + MaxWidth: 0, + }, + expectedH: 500, + expectedW: 500, + }, + // Both extraParams.MaxHeight and extraParams.MaxWidth are greater than 0, but the image size is smaller than the limits + { + extraParams: config.ExtraParams{ + MaxHeight: 1000, + MaxWidth: 1000, + }, + expectedH: 500, + expectedW: 500, + }, + // Both extraParams.MaxHeight and extraParams.MaxWidth are greater than 0, and the image exceeds the limits + { + extraParams: config.ExtraParams{ + MaxHeight: 200, + MaxWidth: 200, + }, + expectedH: 200, + expectedW: 200, + }, + // Only MaxHeight is set to 200 + { + extraParams: config.ExtraParams{ + MaxHeight: 200, + MaxWidth: 0, + }, + expectedH: 200, + expectedW: 200, + }, + + // Test for Width and Height + { + extraParams: config.ExtraParams{ + Width: 200, + Height: 200, + }, + expectedH: 200, + expectedW: 200, + }, + { + extraParams: config.ExtraParams{ + Width: 200, + Height: 500, + }, + expectedH: 500, + expectedW: 200, + }, + } + + // Iterate through the test cases and perform the tests + for _, tc := range testCases { + err := resizeImage(img, tc.extraParams) + if err != nil { + t.Errorf("resizeImage failed with error: %v", err) + } + + // Verify if the adjusted image height and width match the expected values + actualH := img.Height() + actualW := img.Width() + if actualH != tc.expectedH || actualW != tc.expectedW { + t.Errorf("resizeImage failed: expected (%d, %d), got (%d, %d)", tc.expectedH, tc.expectedW, actualH, actualW) + } + } +} diff --git a/encoder/rawconvert.go b/encoder/rawconvert.go new file mode 100644 index 000000000..c55a0133d --- /dev/null +++ b/encoder/rawconvert.go @@ -0,0 +1,22 @@ +package encoder + +import ( + "path/filepath" + + "github.com/jeremytorres/rawparser" +) + +func ConvertRawToJPG(rawPath, optimizedPath string) (string, bool) { + parser, _ := rawparser.NewNefParser(true) + info := &rawparser.RawFileInfo{ + File: rawPath, + Quality: 100, + DestDir: optimizedPath, + } + _, err := parser.ProcessFile(info) + if err == nil { + _, file := filepath.Split(rawPath) + return optimizedPath + file + "_extracted.jpg", true + } + return rawPath, false +} diff --git a/encoder/rawconvert_test.go b/encoder/rawconvert_test.go new file mode 100644 index 000000000..33277dbb7 --- /dev/null +++ b/encoder/rawconvert_test.go @@ -0,0 +1,26 @@ +package encoder + +import ( + "testing" +) + +func TestConvertRawToJPG(t *testing.T) { + testCases := []struct { + rawPath string + optimizedPath string + expectedResult string + expectedStatus bool + }{ + // blackbird.NEF is from https://github.com/jewright/nef-to-jpg/blob/main/photoconverter/Sample-Images/blackbird.NEF + {"../pics/blackbird.NEF", "../exhaust_test/", "../exhaust_test/blackbird.NEF_extracted.jpg", true}, + {"../pics/big.jpg", "../exhaust_test/", "../pics/big.jpg", false}, + } + + for _, tc := range testCases { + result, status := ConvertRawToJPG(tc.rawPath, tc.optimizedPath) + + if result != tc.expectedResult || status != tc.expectedStatus { + t.Errorf("ConvertRawToJPG(%s, %s) => (%s, %t), expected (%s, %t)", tc.rawPath, tc.optimizedPath, result, status, tc.expectedResult, tc.expectedStatus) + } + } +} diff --git a/go.mod b/go.mod index 036e519a6..bcd37f44e 100644 --- a/go.mod +++ b/go.mod @@ -1,37 +1,42 @@ module webp_server_go -go 1.20 +go 1.23 require ( + github.com/buckket/go-blurhash v1.1.0 github.com/cespare/xxhash v1.1.0 - github.com/davidbyttow/govips/v2 v2.13.0 - github.com/gofiber/fiber/v2 v2.48.0 - github.com/h2non/filetype v1.1.3 + github.com/davidbyttow/govips/v2 v2.15.0 + github.com/gofiber/fiber/v2 v2.52.6 + github.com/h2non/filetype v1.1.4-0.20230123234534-cfcd7d097bc4 + github.com/h2non/go-is-svg v0.0.0-20160927212452-35e8c4b0612c + github.com/jeremytorres/rawparser v1.0.2 + github.com/mileusna/useragent v1.3.5 github.com/patrickmn/go-cache v2.1.0+incompatible - github.com/schollz/progressbar/v3 v3.13.1 + github.com/schollz/progressbar/v3 v3.18.0 github.com/sirupsen/logrus v1.9.3 - github.com/stretchr/testify v1.8.4 - github.com/valyala/fasthttp v1.48.0 + github.com/stretchr/testify v1.10.0 + github.com/valyala/fasthttp v1.58.0 ) require ( - github.com/andybalholm/brotli v1.0.5 // indirect + github.com/andybalholm/brotli v1.1.1 // indirect github.com/davecgh/go-spew v1.1.1 // indirect - github.com/google/uuid v1.3.0 // indirect - github.com/h2non/go-is-svg v0.0.0-20160927212452-35e8c4b0612c // indirect - github.com/klauspost/compress v1.16.3 // indirect + github.com/google/uuid v1.6.0 // indirect + github.com/klauspost/compress v1.17.11 // indirect github.com/mattn/go-colorable v0.1.13 // indirect - github.com/mattn/go-isatty v0.0.19 // indirect - github.com/mattn/go-runewidth v0.0.14 // indirect + github.com/mattn/go-isatty v0.0.20 // indirect + github.com/mattn/go-runewidth v0.0.16 // indirect github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db // indirect github.com/pmezard/go-difflib v1.0.0 // indirect - github.com/rivo/uniseg v0.2.0 // indirect + github.com/rivo/uniseg v0.4.7 // indirect github.com/valyala/bytebufferpool v1.0.0 // indirect github.com/valyala/tcplisten v1.0.0 // indirect - golang.org/x/image v0.5.0 // indirect - golang.org/x/net v0.8.0 // indirect - golang.org/x/sys v0.10.0 // indirect - golang.org/x/term v0.6.0 // indirect - golang.org/x/text v0.8.0 // indirect + golang.org/x/image v0.18.0 // indirect + golang.org/x/net v0.33.0 // indirect + golang.org/x/sys v0.29.0 // indirect + golang.org/x/term v0.28.0 // indirect + golang.org/x/text v0.21.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) + +replace github.com/jeremytorres/rawparser v1.0.2 => github.com/webp-sh/rawparser v0.0.0-20240311121240-15117cd3320a diff --git a/go.sum b/go.sum index a8fb305cc..322357619 100644 --- a/go.sum +++ b/go.sum @@ -1,36 +1,40 @@ github.com/OneOfOne/xxhash v1.2.2 h1:KMrpdQIwFcEqXDklaen+P1axHaj9BSKzvpUUfnHldSE= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= -github.com/andybalholm/brotli v1.0.5 h1:8uQZIdzKmjc/iuPu7O2ioW48L81FgatrcpfFmiq/cCs= -github.com/andybalholm/brotli v1.0.5/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig= +github.com/andybalholm/brotli v1.1.1 h1:PR2pgnyFznKEugtsUo0xLdDop5SKXd5Qf5ysW+7XdTA= +github.com/andybalholm/brotli v1.1.1/go.mod h1:05ib4cKhjx3OQYUY22hTVd34Bc8upXjOLL2rKwwZBoA= +github.com/buckket/go-blurhash v1.1.0 h1:X5M6r0LIvwdvKiUtiNcRL2YlmOfMzYobI3VCKCZc9Do= +github.com/buckket/go-blurhash v1.1.0/go.mod h1:aT2iqo5W9vu9GpyoLErKfTHwgODsZp3bQfXjXJUxNb8= github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= +github.com/chengxilo/virtualterm v1.0.4 h1:Z6IpERbRVlfB8WkOmtbHiDbBANU7cimRIof7mk9/PwM= +github.com/chengxilo/virtualterm v1.0.4/go.mod h1:DyxxBZz/x1iqJjFxTFcr6/x+jSpqN0iwWCOK1q10rlY= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/davidbyttow/govips/v2 v2.13.0 h1:5MK9ZcXZC5GzUR9Ca8fJwOYqMgll/H096ec0PJP59QM= -github.com/davidbyttow/govips/v2 v2.13.0/go.mod h1:LPTrwWtNa5n4yl9UC52YBOEGdZcY5hDTP4Ms2QWasTw= -github.com/gofiber/fiber/v2 v2.48.0 h1:cRVMCb9aUJDsyHxGFLwz/sGzDggdailZZyptU9F9cU0= -github.com/gofiber/fiber/v2 v2.48.0/go.mod h1:xqJgfqrc23FJuqGOW6DVgi3HyZEm2Mn9pRqUb2kHSX8= -github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= -github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/h2non/filetype v1.1.3 h1:FKkx9QbD7HR/zjK1Ia5XiBsq9zdLi5Kf3zGyFTAFkGg= -github.com/h2non/filetype v1.1.3/go.mod h1:319b3zT68BvV+WRj7cwy856M2ehB3HqNOt6sy1HndBY= +github.com/davidbyttow/govips/v2 v2.15.0 h1:h3lF+rQElBzGXbQSSPqmE3XGySPhcQo2x3t5l/dZ+pU= +github.com/davidbyttow/govips/v2 v2.15.0/go.mod h1:3OQCHj0nf5Mnrplh5VlNvmx3IhJXyxbAoTJZPflUjmM= +github.com/gofiber/fiber/v2 v2.52.6 h1:Rfp+ILPiYSvvVuIPvxrBns+HJp8qGLDnLJawAu27XVI= +github.com/gofiber/fiber/v2 v2.52.6/go.mod h1:YEcBbO/FB+5M1IZNBP9FO3J9281zgPAreiI1oqg8nDw= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/h2non/filetype v1.1.4-0.20230123234534-cfcd7d097bc4 h1:k7FGP5I7raiaC3aAzCLddcoxzboIrOm6/FVRXjp/5JM= +github.com/h2non/filetype v1.1.4-0.20230123234534-cfcd7d097bc4/go.mod h1:319b3zT68BvV+WRj7cwy856M2ehB3HqNOt6sy1HndBY= github.com/h2non/go-is-svg v0.0.0-20160927212452-35e8c4b0612c h1:fEE5/5VNnYUoBOj2I9TP8Jc+a7lge3QWn9DKE7NCwfc= github.com/h2non/go-is-svg v0.0.0-20160927212452-35e8c4b0612c/go.mod h1:ObS/W+h8RYb1Y7fYivughjxojTmIu5iAIjSrSLCLeqE= -github.com/k0kubun/go-ansi v0.0.0-20180517002512-3bf9e2903213/go.mod h1:vNUNkEQ1e29fT/6vq2aBdFsgNPmy8qMdSay1npru+Sw= -github.com/klauspost/compress v1.16.3 h1:XuJt9zzcnaz6a16/OU53ZjWp/v7/42WcR5t2a0PcNQY= -github.com/klauspost/compress v1.16.3/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= +github.com/klauspost/compress v1.17.11 h1:In6xLpyWOi1+C7tXUUWv2ot1QvBjxevKAaI6IXrJmUc= +github.com/klauspost/compress v1.17.11/go.mod h1:pMDklpSncoRMuLFrf1W9Ss9KT+0rH90U12bZKk7uwG0= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= -github.com/mattn/go-isatty v0.0.17/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= -github.com/mattn/go-isatty v0.0.19 h1:JITubQf0MOLdlGRuRq+jtsDlekdYPia9ZFsB8h/APPA= -github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= -github.com/mattn/go-runewidth v0.0.14 h1:+xnbZSEeDbOIg5/mE6JF0w6n9duR1l3/WmbinWVwUuU= -github.com/mattn/go-runewidth v0.0.14/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= +github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= +github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/mattn/go-runewidth v0.0.16 h1:E5ScNMtiwvlvB5paMFdw9p4kSQzbXFikJ5SQO6TULQc= +github.com/mattn/go-runewidth v0.0.16/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= +github.com/mileusna/useragent v1.3.5 h1:SJM5NzBmh/hO+4LGeATKpaEX9+b4vcGg2qXGLiNGDws= +github.com/mileusna/useragent v1.3.5/go.mod h1:3d8TOmwL/5I8pJjyVDteHtgDGcefrFUX4ccGOMKNYYc= github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db h1:62I3jR2EmQ4l5rM/4FEfDWcRD+abF5XlKShorW5LRoQ= github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db/go.mod h1:l0dey0ia/Uv7NcFFVbCLtqEBQbrT4OCwCSKTEv6enCw= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs= @@ -39,40 +43,52 @@ github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaR github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/rivo/uniseg v0.2.0 h1:S1pD9weZBuJdFmowNwbpi7BJ8TNftyUImj/0WQi72jY= github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= -github.com/schollz/progressbar/v3 v3.13.1 h1:o8rySDYiQ59Mwzy2FELeHY5ZARXZTVJC7iHD6PEFUiE= -github.com/schollz/progressbar/v3 v3.13.1/go.mod h1:xvrbki8kfT1fzWzBT/UZd9L6GA+jdL7HAgq2RFnO6fQ= +github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ= +github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= +github.com/schollz/progressbar/v3 v3.18.0 h1:uXdoHABRFmNIjUfte/Ex7WtuyVslrw2wVPQmCN62HpA= +github.com/schollz/progressbar/v3 v3.18.0/go.mod h1:IsO3lpbaGuzh8zIMzgY3+J8l4C8GjO0Y9S69eFvNsec= github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72 h1:qLC7fQah7D6K1B0ujays3HV9gkFtllcxhzImRR7ArPQ= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= -github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= +github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= -github.com/valyala/fasthttp v1.48.0 h1:oJWvHb9BIZToTQS3MuQ2R3bJZiNSa2KiNdeI8A+79Tc= -github.com/valyala/fasthttp v1.48.0/go.mod h1:k2zXd82h/7UZc3VOdJ2WaUqt1uZ/XpXAfE9i+HBC3lA= +github.com/valyala/fasthttp v1.58.0 h1:GGB2dWxSbEprU9j0iMJHgdKYJVDyjrOwF9RE59PbRuE= +github.com/valyala/fasthttp v1.58.0/go.mod h1:SYXvHHaFp7QZHGKSHmoMipInhrI5StHrhDTYVEjK/Kw= github.com/valyala/tcplisten v1.0.0 h1:rBHj/Xf+E1tRGZyWIWwJDiRY0zc1Js+CV5DqwacVSA8= github.com/valyala/tcplisten v1.0.0/go.mod h1:T0xQ8SeCZGxckz9qRXTfG43PvQ/mcWh7FwZEA7Ioqkc= +github.com/webp-sh/rawparser v0.0.0-20240311121240-15117cd3320a h1:yFNUYbDL81wQZ7AQmBhkS+ZDfTugwepVI4LUQ/tQBAc= +github.com/webp-sh/rawparser v0.0.0-20240311121240-15117cd3320a/go.mod h1:X0j2dOqH3ecGRuWvkThgDy+NKAfIwSN9wAOQlMcFOfY= +github.com/xyproto/randomstring v1.0.5 h1:YtlWPoRdgMu3NZtP45drfy1GKoojuR7hmRcnhZqKjWU= +github.com/xyproto/randomstring v1.0.5/go.mod h1:rgmS5DeNXLivK7YprL0pY+lTuhNQW3iGxZ18UQApw/E= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/image v0.5.0 h1:5JMiNunQeQw++mMOz48/ISeNu3Iweh/JaZU8ZLqHRrI= -golang.org/x/image v0.5.0/go.mod h1:FVC7BI/5Ym8R25iw5OLsgshdUBbT1h5jZTpA+mvAdZ4= +golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU= +golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs= +golang.org/x/image v0.10.0/go.mod h1:jtrku+n79PfroUbvDdeUWMAI+heR786BofxrbiSF+J0= +golang.org/x/image v0.18.0 h1:jGzIakQa/ZXI1I0Fxvaa9W7yP25TqT6cHIHn+6CqvSQ= +golang.org/x/image v0.18.0/go.mod h1:4yyo5vMFQjVjUcVk4jEQcU9MGy/rulF5WvUILseCM2E= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= +golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= -golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= -golang.org/x/net v0.8.0 h1:Zrh2ngAOFYneWTAIAPethzeaQLuHwhuBkuV6ZiRnUaQ= -golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= +golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= +golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= +golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= +golang.org/x/net v0.23.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg= +golang.org/x/net v0.33.0 h1:74SYHlV8BIgHIFC/LrYkOGIwL19eTYXQ5wc6TBuO36I= +golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -82,22 +98,32 @@ golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.10.0 h1:SqMFp9UcQJZa+pmYuAKjd9xq1f0j5rLcDIk0mj4qAsA= -golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.29.0 h1:TPYlXGxvx1MGTn2GiZDhnjPA9wZzZeGKHHmKhHYvgaU= +golang.org/x/sys v0.29.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= -golang.org/x/term v0.6.0 h1:clScbb1cHjoCkyRbWwBEUZ5H/tIFu5TAXIqaZD0Gcjw= -golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U= +golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= +golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk= +golang.org/x/term v0.18.0/go.mod h1:ILwASektA3OnRv7amZ1xhE/KTR+u50pbXfZ03+6Nx58= +golang.org/x/term v0.28.0 h1:/Ts8HFuMR2E6IP/jlo7QVLZHggjKQbhu/7H0LJFr3Gg= +golang.org/x/term v0.28.0/go.mod h1:Sw/lC2IAUZ92udQNf3WodGtn4k/XoLyZoh8v/8uiwek= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= -golang.org/x/text v0.8.0 h1:57P1ETyNKtuIjB4SRd15iJxuhj8Gc416Y78H3qgMh68= -golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= +golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= +golang.org/x/text v0.11.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= +golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/text v0.21.0 h1:zyQAAkrwaneQ066sspRyJaG9VNi/YJ1NfzcGB3hZ/qo= +golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= +golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20200902074654-038fdea0a05b h1:QRR6H1YWRnHb4Y/HeNFCTJLFVxaq6wH4YuVdsUOr75U= diff --git a/handler/healthz.go b/handler/healthz.go new file mode 100644 index 000000000..30c0cde9a --- /dev/null +++ b/handler/healthz.go @@ -0,0 +1,9 @@ +package handler + +import ( + "github.com/gofiber/fiber/v2" +) + +func Healthz(c *fiber.Ctx) error { + return c.SendString("WebP Server Go up and running!πŸ₯³") +} diff --git a/handler/remote.go b/handler/remote.go index 66f9665db..a8ee679fc 100644 --- a/handler/remote.go +++ b/handler/remote.go @@ -12,6 +12,7 @@ import ( "github.com/gofiber/fiber/v2" "github.com/h2non/filetype" + "github.com/patrickmn/go-cache" log "github.com/sirupsen/logrus" ) @@ -75,18 +76,45 @@ func downloadFile(filepath string, url string) { } -func fetchRemoteImg(url string) config.MetaFile { +func fetchRemoteImg(url string, subdir string) config.MetaFile { // url is https://test.webp.sh/mypic/123.jpg?someother=200&somebugs=200 // How do we know if the remote img is changed? we're using hash(etag+length) - log.Infof("Remote Addr is %s, pinging for info...", url) - etag := pingURL(url) - metadata := helper.ReadMetadata(url, etag) - localRawImagePath := path.Join(config.RemoteRaw, metadata.Id) + var etag string + + cacheKey := subdir + ":" + helper.HashString(url) + + if val, found := config.RemoteCache.Get(cacheKey); found { + if etagVal, ok := val.(string); ok { + log.Infof("Using cache for remote addr: %s", url) + etag = etagVal + } else { + config.RemoteCache.Delete(cacheKey) + } + } + + if etag == "" { + log.Infof("Remote Addr is %s, pinging for info...", url) + etag = pingURL(url) + if etag != "" { + config.RemoteCache.Set(cacheKey, etag, cache.DefaultExpiration) + } + } + + metadata := helper.ReadMetadata(url, etag, subdir) + localRawImagePath := path.Join(config.Config.RemoteRawPath, subdir, metadata.Id) + localExhaustImagePath := path.Join(config.Config.ExhaustPath, subdir, metadata.Id) if !helper.ImageExists(localRawImagePath) || metadata.Checksum != helper.HashString(etag) { - // remote file has changed or local file not exists - log.Info("Remote file not found in remote-raw, re-fetching...") - cleanProxyCache(path.Join(config.Config.ExhaustPath, metadata.Id+"*")) + cleanProxyCache(localExhaustImagePath) + if metadata.Checksum != helper.HashString(etag) { + // remote file has changed + log.Info("Remote file changed, updating metadata and fetching image source...") + helper.DeleteMetadata(url, subdir) + helper.WriteMetadata(url, etag, subdir) + } else { + // local file not exists + log.Info("Remote file not found in remote-raw, re-fetching...") + } downloadFile(localRawImagePath, url) } return metadata diff --git a/handler/router.go b/handler/router.go index 71d1fd910..8ee068d2f 100644 --- a/handler/router.go +++ b/handler/router.go @@ -3,6 +3,9 @@ package handler import ( "net/http" "net/url" + "regexp" + "slices" + "strings" "webp_server_go/config" "webp_server_go/encoder" "webp_server_go/helper" @@ -20,56 +23,158 @@ func Convert(c *fiber.Ctx) error { // 2. generate rawImagePath, could be local path or remote url(possible with query string) // 3. pass it to encoder, get the result, send it back + // normal http request will start with / + if !strings.HasPrefix(c.Path(), "/") { + _ = c.SendStatus(http.StatusBadRequest) + return nil + } + var ( - reqURI, _ = url.QueryUnescape(c.Path()) // /mypic/123.jpg - reqURIwithQuery, _ = url.QueryUnescape(c.OriginalURL()) // /mypic/123.jpg?someother=200&somebugs=200 - filename = path.Base(reqURI) + reqHostname = c.Hostname() + reqHost = c.Protocol() + "://" + reqHostname // http://www.example.com:8000 + reqHeader = &c.Request().Header + + reqURIRaw, _ = url.QueryUnescape(c.Path()) // /mypic/123.jpg + reqURIwithQueryRaw, _ = url.QueryUnescape(c.OriginalURL()) // /mypic/123.jpg?someother=200&somebugs=200 + reqURI = path.Clean(reqURIRaw) // delete ../ in reqURI to mitigate directory traversal + reqURIwithQuery = path.Clean(reqURIwithQueryRaw) // Sometimes reqURIwithQuery can be https://example.tld/mypic/123.jpg?someother=200&somebugs=200, we need to extract it + + filename = path.Base(reqURI) + realRemoteAddr = "" + targetHostName = config.LocalHostAlias + targetHost = config.Config.ImgPath + proxyMode = config.ProxyMode + mapMode = false + + meta = c.Query("meta") // Meta request + + width, _ = strconv.Atoi(c.Query("width")) // Extra Params + height, _ = strconv.Atoi(c.Query("height")) // Extra Params + maxHeight, _ = strconv.Atoi(c.Query("max_height")) // Extra Params + maxWidth, _ = strconv.Atoi(c.Query("max_width")) // Extra Params + extraParams = config.ExtraParams{ + Width: width, + Height: height, + MaxWidth: maxWidth, + MaxHeight: maxHeight, + } ) - if !helper.CheckAllowedType(filename) { + log.Debugf("Incoming connection from %s %s %s", c.IP(), reqHostname, reqURIwithQuery) + + if !helper.CheckAllowedExtension(filename) { msg := "File extension not allowed! " + filename log.Warn(msg) c.Status(http.StatusBadRequest) - _ = c.Send([]byte(msg)) + _ = c.SendString(msg) return nil } - // Sometimes reqURIwithQuery can be https://example.tld/mypic/123.jpg?someother=200&somebugs=200, we need to extract it. - // delete ../ in reqURI to mitigate directory traversal - reqURI = path.Clean(reqURI) - reqURIwithQuery = path.Clean(reqURIwithQuery) + // Check if the file extension is allowed and not with image extension + // In this case we will serve the file directly + // Since here we've already sent non-image file, "raw" is not supported by default in the following code + if helper.CheckAllowedExtension(filename) && !helper.CheckImageExtension(filename) { + return c.SendFile(path.Join(config.Config.ImgPath, reqURI)) + } + + // Rewrite the target backend if a mapping rule matches the hostname + if hostMap, hostMapFound := config.Config.ImageMap[reqHost]; hostMapFound { + log.Debugf("Found host mapping %s -> %s", reqHostname, hostMap) + targetHostUrl, _ := url.Parse(hostMap) + targetHostName = targetHostUrl.Host + targetHost = targetHostUrl.Scheme + "://" + targetHostUrl.Host + proxyMode = true + } else { + // There's not matching host mapping, now check for any URI map that apply + httpRegexpMatcher := regexp.MustCompile(config.HttpRegexp) + for uriMap, uriMapTarget := range config.Config.ImageMap { + if strings.HasPrefix(reqURI, uriMap) { + log.Debugf("Found URI mapping %s -> %s", uriMap, uriMapTarget) + mapMode = true + + // if uriMapTarget we use the proxy mode to fetch the remote + if httpRegexpMatcher.Match([]byte(uriMapTarget)) { + targetHostUrl, _ := url.Parse(uriMapTarget) + targetHostName = targetHostUrl.Host + targetHost = targetHostUrl.Scheme + "://" + targetHostUrl.Host + reqURI = strings.Replace(reqURI, uriMap, targetHostUrl.Path, 1) + reqURIwithQuery = strings.Replace(reqURIwithQuery, uriMap, targetHostUrl.Path, 1) + proxyMode = true + } else { + reqURI = strings.Replace(reqURI, uriMap, uriMapTarget, 1) + reqURIwithQuery = strings.Replace(reqURIwithQuery, uriMap, uriMapTarget, 1) + } + break + } + } + + } + + if proxyMode { + + if !mapMode { + // Don't deal with the encoding to avoid upstream compatibilities + reqURI = c.Path() + reqURIwithQuery = c.OriginalURL() + } - width, _ := strconv.Atoi(c.Query("width")) - height, _ := strconv.Atoi(c.Query("height")) + log.Tracef("reqURIwithQuery is %s", reqURIwithQuery) - var extraParams = config.ExtraParams{ - Width: width, - Height: height, + // Replace host in the URL + // realRemoteAddr = strings.Replace(reqURIwithQuery, reqHost, targetHost, 1) + realRemoteAddr = targetHost + reqURIwithQuery + log.Debugf("realRemoteAddr is %s", realRemoteAddr) } var rawImageAbs string var metadata = config.MetaFile{} - if config.ProxyMode { + if proxyMode { // this is proxyMode, we'll have to use this url to download and save it to local path, which also gives us rawImageAbs // https://test.webp.sh/mypic/123.jpg?someother=200&somebugs=200 - metadata = fetchRemoteImg(config.Config.ImgPath + reqURIwithQuery) - rawImageAbs = path.Join(config.RemoteRaw, metadata.Id) + + metadata = fetchRemoteImg(realRemoteAddr, targetHostName) + rawImageAbs = path.Join(config.Config.RemoteRawPath, targetHostName, metadata.Id) } else { // not proxyMode, we'll use local path - metadata = helper.ReadMetadata(reqURIwithQuery, "") - rawImageAbs = path.Join(config.Config.ImgPath, reqURI) + metadata = helper.ReadMetadata(reqURIwithQuery, "", targetHostName) + if !mapMode { + // by default images are hosted in ImgPath + rawImageAbs = path.Join(config.Config.ImgPath, reqURI) + } else { + rawImageAbs = reqURI + } // detect if source file has changed if metadata.Checksum != helper.HashFile(rawImageAbs) { log.Info("Source file has changed, re-encoding...") - helper.WriteMetadata(reqURIwithQuery, "") - cleanProxyCache(path.Join(config.Config.ExhaustPath, metadata.Id)) + helper.WriteMetadata(reqURIwithQuery, "", targetHostName) + cleanProxyCache(path.Join(config.Config.ExhaustPath, targetHostName, metadata.Id)) } } - goodFormat := helper.GuessSupportedFormat(&c.Request().Header) - // resize itself and return if only one format(raw) is supported - if len(goodFormat) == 1 { - dest := path.Join(config.Config.ExhaustPath, metadata.Id) + // If meta request, return the metadata + if meta == "full" { + return c.JSON(fiber.Map{ + "height": metadata.ImageMeta.Height, + "width": metadata.ImageMeta.Width, + "size": metadata.ImageMeta.Size, + "format": metadata.ImageMeta.Format, + "colorspace": metadata.ImageMeta.Colorspace, + "num_pages": metadata.ImageMeta.NumPages, + "blurhash": metadata.ImageMeta.Blurhash, + }) + } + + supportedFormats := helper.GuessSupportedFormat(reqHeader) + // resize itself and return if only raw(jpg,jpeg,png,gif) is supported + if supportedFormats["jpg"] == true && + supportedFormats["jpeg"] == true && + supportedFormats["png"] == true && + supportedFormats["gif"] == true && + supportedFormats["webp"] == false && + supportedFormats["avif"] == false && + supportedFormats["jxl"] == false && + supportedFormats["heic"] == false { + dest := path.Join(config.Config.ExhaustPath, targetHostName, metadata.Id) if !helper.ImageExists(dest) { encoder.ResizeItself(rawImageAbs, dest, extraParams) } @@ -78,24 +183,31 @@ func Convert(c *fiber.Ctx) error { // Check the original image for existence, if !helper.ImageExists(rawImageAbs) { - msg := "image not found" + helper.DeleteMetadata(reqURIwithQuery, targetHostName) + msg := "Image not found!" _ = c.Send([]byte(msg)) log.Warn(msg) _ = c.SendStatus(404) return nil } - avifAbs, webpAbs := helper.GenOptimizedAbsPath(metadata) - encoder.ConvertFilter(rawImageAbs, avifAbs, webpAbs, extraParams, nil) + avifAbs, webpAbs, jxlAbs := helper.GenOptimizedAbsPath(metadata, targetHostName) + // Do the convertion based on supported formats and config + encoder.ConvertFilter(rawImageAbs, jxlAbs, avifAbs, webpAbs, extraParams, supportedFormats, nil) - var availableFiles = []string{rawImageAbs} - for _, v := range goodFormat { - if v == "avif" { - availableFiles = append(availableFiles, avifAbs) - } - if v == "webp" { - availableFiles = append(availableFiles, webpAbs) - } + var availableFiles = []string{} + // If source image is in jpg/jpeg/png/gif, we can add it to the available files + if slices.Contains([]string{"jpg", "jpeg", "png", "gif"}, helper.GetImageExtension(rawImageAbs)) { + availableFiles = append(availableFiles, rawImageAbs) + } + if supportedFormats["avif"] { + availableFiles = append(availableFiles, avifAbs) + } + if supportedFormats["webp"] { + availableFiles = append(availableFiles, webpAbs) + } + if supportedFormats["jxl"] { + availableFiles = append(availableFiles, jxlAbs) } finalFilename := helper.FindSmallestFiles(availableFiles) diff --git a/handler/router_test.go b/handler/router_test.go new file mode 100644 index 000000000..a46900bef --- /dev/null +++ b/handler/router_test.go @@ -0,0 +1,372 @@ +package handler + +import ( + "io" + "net/http" + "net/http/httptest" + "net/url" + "os" + "testing" + "time" + "webp_server_go/config" + "webp_server_go/helper" + + "github.com/gofiber/fiber/v2" + "github.com/gofiber/fiber/v2/middleware/etag" + "github.com/patrickmn/go-cache" + "github.com/stretchr/testify/assert" +) + +var ( + chromeUA = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.183 Safari/537.36" + safariUA = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_4) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/13.1 Safari/605.1.15" + safari17UA = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.3.1 Safari/605.1.15" // <- Mac with Safari 17 + curlUA = "curl/7.64.1" + + acceptWebP = "image/webp,image/apng,image/*,*/*;q=0.8" + acceptAvif = "image/avif,image/*,*/*;q=0.8" + acceptLegacy = "image/jpeg,image/png" +) + +func setupParam() { + // setup parameters here... + config.Config.ImgPath = "../pics" + config.Config.ExhaustPath = "../exhaust_test" + config.Config.AllowedTypes = []string{"jpg", "png", "jpeg", "bmp", "heic", "avif"} + config.Config.MetadataPath = "../metadata" + config.Config.RemoteRawPath = "../remote-raw" + config.ProxyMode = false + config.Config.EnableWebP = true + config.Config.EnableAVIF = false + config.Config.Quality = 80 + config.Config.ImageMap = map[string]string{} + config.RemoteCache = cache.New(cache.NoExpiration, 10*time.Minute) +} + +func requestToServer(reqUrl string, app *fiber.App, ua, accept string) (*http.Response, []byte) { + parsedUrl, _ := url.Parse(reqUrl) + req := httptest.NewRequest("GET", parsedUrl.EscapedPath(), nil) + req.Header.Set("User-Agent", ua) + req.Header.Set("Accept", accept) + req.Header.Set("Host", parsedUrl.Host) + req.Host = parsedUrl.Host + resp, err := app.Test(req, 120000) + if err != nil { + return nil, nil + } + data, _ := io.ReadAll(resp.Body) + return resp, data +} + +func TestServerHeaders(t *testing.T) { + setupParam() + var app = fiber.New() + app.Use(etag.New(etag.Config{ + Weak: true, + })) + app.Get("/*", Convert) + url := "http://127.0.0.1:3333/webp_server.bmp" + + // test for chrome + response, _ := requestToServer(url, app, chromeUA, acceptWebP) + defer response.Body.Close() + ratio := response.Header.Get("X-Compression-Rate") + etag := response.Header.Get("Etag") + + assert.NotEqual(t, "", ratio) + assert.NotEqual(t, "", etag) + + // test for safari + response, _ = requestToServer(url, app, safariUA, acceptLegacy) + defer response.Body.Close() + // ratio = response.Header.Get("X-Compression-Rate") + etag = response.Header.Get("Etag") + + assert.NotEqual(t, "", etag) +} + +func TestConvertDuplicates(t *testing.T) { + setupParam() + N := 3 + + var testLink = map[string]string{ + "http://127.0.0.1:3333/webp_server.jpg": "image/webp", + "http://127.0.0.1:3333/webp_server.bmp": "image/webp", + "http://127.0.0.1:3333/webp_server.png": "image/webp", + "http://127.0.0.1:3333/empty.jpg": "", + "http://127.0.0.1:3333/png.jpg": "image/webp", + "http://127.0.0.1:3333/12314.jpg": "", + "http://127.0.0.1:3333/dir1/inside.jpg": "image/webp", + "http://127.0.0.1:3333/%e5%a4%aa%e7%a5%9e%e5%95%a6.png": "image/webp", + "http://127.0.0.1:3333/ε€ͺη₯žε•¦.png": "image/webp", + } + + var app = fiber.New() + app.Get("/*", Convert) + + // test Chrome + for url, respType := range testLink { + for range N { + resp, data := requestToServer(url, app, chromeUA, acceptWebP) + defer resp.Body.Close() + contentType := helper.GetContentType(data) + assert.Equal(t, respType, contentType) + } + } + +} +func TestConvert(t *testing.T) { + setupParam() + // TODO: old-style test, better update it with accept headers + var testChromeLink = map[string]string{ + "http://127.0.0.1:3333/webp_server.jpg": "image/webp", + "http://127.0.0.1:3333/webp_server.bmp": "image/webp", + "http://127.0.0.1:3333/webp_server.png": "image/webp", + "http://127.0.0.1:3333/empty.jpg": "", + "http://127.0.0.1:3333/png.jpg": "image/webp", + "http://127.0.0.1:3333/12314.jpg": "", + "http://127.0.0.1:3333/dir1/inside.jpg": "image/webp", + "http://127.0.0.1:3333/%e5%a4%aa%e7%a5%9e%e5%95%a6.png": "image/webp", + "http://127.0.0.1:3333/ε€ͺη₯žε•¦.png": "image/webp", + // Source: https://filesamples.com/formats/heic + "http://127.0.0.1:3333/sample3.heic": "image/webp", // webp because browser does not support heic + // Source: https://raw.githubusercontent.com/link-u/avif-sample-images/refs/heads/master/kimono.avif + "http://127.0.0.1:3333/kimono.avif": "image/webp", // webp because browser does not support avif + } + + var testChromeAvifLink = map[string]string{ + "http://127.0.0.1:3333/webp_server.jpg": "image/avif", + "http://127.0.0.1:3333/webp_server.bmp": "image/avif", + "http://127.0.0.1:3333/webp_server.png": "image/avif", + "http://127.0.0.1:3333/empty.jpg": "", + "http://127.0.0.1:3333/png.jpg": "image/avif", + "http://127.0.0.1:3333/12314.jpg": "", + "http://127.0.0.1:3333/dir1/inside.jpg": "image/avif", + "http://127.0.0.1:3333/%e5%a4%aa%e7%a5%9e%e5%95%a6.png": "image/avif", + "http://127.0.0.1:3333/ε€ͺη₯žε•¦.png": "image/avif", + } + + var testSafariLink = map[string]string{ + "http://127.0.0.1:3333/webp_server.jpg": "image/jpeg", + "http://127.0.0.1:3333/webp_server.bmp": "image/png", // png instead oft bmp because ResizeItself() uses ExportNative() + "http://127.0.0.1:3333/webp_server.png": "image/png", + "http://127.0.0.1:3333/empty.jpg": "", + "http://127.0.0.1:3333/png.jpg": "image/png", + "http://127.0.0.1:3333/12314.jpg": "", + "http://127.0.0.1:3333/dir1/inside.jpg": "image/jpeg", + } + + var app = fiber.New() + app.Get("/*", Convert) + + // // test Chrome + for url, respType := range testChromeLink { + resp, data := requestToServer(url, app, chromeUA, acceptWebP) + defer resp.Body.Close() + contentType := helper.GetContentType(data) + assert.Equal(t, respType, contentType) + } + + // test Safari + for url, respType := range testSafariLink { + resp, data := requestToServer(url, app, safariUA, acceptLegacy) + defer resp.Body.Close() + contentType := helper.GetContentType(data) + assert.Equal(t, respType, contentType) + } + + // test Avif is processed in proxy mode + config.Config.EnableAVIF = true + for url, respType := range testChromeAvifLink { + resp, data := requestToServer(url, app, chromeUA, acceptAvif) + defer resp.Body.Close() + contentType := helper.GetContentType(data) + assert.NotNil(t, respType) + assert.Equal(t, respType, contentType) + } +} + +func TestConvertNotAllowed(t *testing.T) { + setupParam() + config.Config.AllowedTypes = []string{"jpg", "png", "jpeg"} + + var app = fiber.New() + app.Get("/*", Convert) + + // not allowed, but we have the file, this should return File extension not allowed + url := "http://127.0.0.1:3333/webp_server.bmp" + resp, data := requestToServer(url, app, chromeUA, acceptWebP) + defer resp.Body.Close() + assert.Contains(t, string(data), "File extension not allowed") + + // not allowed, but we have the file, this should return File extension not allowed + url = "http://127.0.0.1:3333/config.json" + resp, data = requestToServer(url, app, chromeUA, acceptWebP) + defer resp.Body.Close() + assert.Contains(t, string(data), "File extension not allowed") + + // not allowed, random file + url = url + "hagdgd" + resp, data = requestToServer(url, app, chromeUA, acceptWebP) + defer resp.Body.Close() + assert.Contains(t, string(data), "File extension not allowed") +} + +func TestConvertPassThrough(t *testing.T) { + setupParam() + config.Config.AllowedTypes = []string{"*"} + + var app = fiber.New() + app.Get("/*", Convert) + + // not allowed, but we have the file, this should return File extension not allowed + url := "http://127.0.0.1:3333/config.json" + resp, data := requestToServer(url, app, chromeUA, acceptWebP) + defer resp.Body.Close() + assert.Equal(t, http.StatusOK, resp.StatusCode) + assert.Equal(t, "application/json", resp.Header.Get("Content-Type")) + assert.Contains(t, string(data), "HOST") +} + +func TestConvertProxyModeBad(t *testing.T) { + setupParam() + config.ProxyMode = true + + var app = fiber.New() + app.Get("/*", Convert) + + // this is local random image, should be 404 + url := "http://127.0.0.1:3333/webp_8888server.bmp" + resp, _ := requestToServer(url, app, chromeUA, acceptWebP) + defer resp.Body.Close() + assert.Equal(t, http.StatusNotFound, resp.StatusCode) + + // this is local random image, test using cURL, should be 404, ref: https://github.com/webp-sh/webp_server_go/issues/197 + resp1, _ := requestToServer(url, app, curlUA, acceptWebP) + defer resp1.Body.Close() + assert.Equal(t, http.StatusNotFound, resp1.StatusCode) + +} + +func TestConvertProxyModeWork(t *testing.T) { + setupParam() + config.ProxyMode = true + config.Config.ImgPath = "https://docs.webp.sh" + + var app = fiber.New() + app.Get("/*", Convert) + + url := "http://127.0.0.1:3333/images/webp_server.jpg" + + resp, data := requestToServer(url, app, chromeUA, acceptWebP) + defer resp.Body.Close() + assert.Equal(t, http.StatusOK, resp.StatusCode) + assert.Equal(t, "image/webp", helper.GetContentType(data)) + + // test proxyMode with Safari + resp, data = requestToServer(url, app, safariUA, acceptLegacy) + defer resp.Body.Close() + assert.Equal(t, http.StatusOK, resp.StatusCode) + assert.Equal(t, "image/jpeg", helper.GetContentType(data)) +} + +func TestConvertProxyImgMap(t *testing.T) { + setupParam() + config.ProxyMode = false + config.Config.ImageMap = map[string]string{ + "/2": "../pics/dir1", + "/3": "../pics3", // Invalid path, does not exists + "www.invalid-path.com": "https://docs.webp.sh", // Invalid, it does not start with '/' + "/www.weird-path.com": "https://docs.webp.sh", + "/www.even-more-werid-path.com": "https://docs.webp.sh/images", + "http://example.com": "https://docs.webp.sh", + } + + var app = fiber.New() + app.Get("/*", Convert) + + var testUrls = map[string]string{ + "http://127.0.0.1:3333/webp_server.jpg": "image/webp", + "http://127.0.0.1:3333/2/inside.jpg": "image/webp", + "http://127.0.0.1:3333/www.weird-path.com/images/webp_server.jpg": "image/webp", + "http://127.0.0.1:3333/www.even-more-werid-path.com/webp_server.jpg": "image/webp", + "http://example.com//images/webp_server.jpg": "image/webp", + } + + var testUrlsLegacy = map[string]string{ + "http://127.0.0.1:3333/webp_server.jpg": "image/jpeg", + "http://127.0.0.1:3333/2/inside.jpg": "image/jpeg", + "http://example.com/images/webp_server.jpg": "image/jpeg", + } + + var testUrlsInvalid = map[string]string{ + "http://127.0.0.1:3333/3/does-not-exist.jpg": "", // Dir mapped does not exist + "http://127.0.0.1:3333/www.weird-path.com/cover.jpg": "", // Host mapped, final URI invalid + } + + for url, respType := range testUrls { + resp, data := requestToServer(url, app, chromeUA, acceptWebP) + defer resp.Body.Close() + assert.Equal(t, http.StatusOK, resp.StatusCode) + assert.Equal(t, respType, helper.GetContentType(data)) + } + + // tests with Safari + for url, respType := range testUrlsLegacy { + resp, data := requestToServer(url, app, safariUA, acceptLegacy) + defer resp.Body.Close() + assert.Equal(t, http.StatusOK, resp.StatusCode) + assert.Equal(t, respType, helper.GetContentType(data)) + } + + for url, respType := range testUrlsInvalid { + resp, data := requestToServer(url, app, safariUA, acceptLegacy) + defer resp.Body.Close() + assert.Equal(t, http.StatusNotFound, resp.StatusCode) + assert.Equal(t, respType, helper.GetContentType(data)) + } +} + +func TestConvertProxyImgMapCWD(t *testing.T) { + setupParam() + config.ProxyMode = false + config.Config.ImgPath = ".." // equivalent to "" when not testing + config.Config.ImageMap = map[string]string{ + "/1": "../pics/dir1", + "/2": "../pics", + "/3": "../pics", // Invalid path, does not exists + "http://www.example.com": "https://docs.webp.sh", + } + + var app = fiber.New() + app.Get("/*", Convert) + + var testUrls = map[string]string{ + "http://127.0.0.1:3333/1/inside.jpg": "image/webp", + "http://127.0.0.1:3333/2/webp_server.jpg": "image/webp", + "http://127.0.0.1:3333/3/webp_server.jpg": "image/webp", + "http://www.example.com/images/webp_server.jpg": "image/webp", + } + + for url, respType := range testUrls { + resp, data := requestToServer(url, app, chromeUA, acceptWebP) + defer resp.Body.Close() + assert.Equal(t, http.StatusOK, resp.StatusCode) + assert.Equal(t, respType, helper.GetContentType(data)) + } +} + +func TestConvertBigger(t *testing.T) { + setupParam() + config.Config.Quality = 100 + + var app = fiber.New() + app.Get("/*", Convert) + + url := "http://127.0.0.1:3333/big.jpg" + resp, data := requestToServer(url, app, chromeUA, acceptWebP) + defer resp.Body.Close() + assert.Equal(t, "image/jpeg", resp.Header.Get("content-type")) + assert.Equal(t, "image/jpeg", helper.GetContentType(data)) + _ = os.RemoveAll(config.Config.ExhaustPath) +} diff --git a/helper/helper.go b/helper/helper.go index 5d81c1d29..d26cbd9cf 100644 --- a/helper/helper.go +++ b/helper/helper.go @@ -9,7 +9,11 @@ import ( "time" "webp_server_go/config" + "slices" + + "github.com/davidbyttow/govips/v2/vips" "github.com/h2non/filetype" + "github.com/mileusna/useragent" "github.com/cespare/xxhash" "github.com/valyala/fasthttp" @@ -18,6 +22,11 @@ import ( log "github.com/sirupsen/logrus" ) +var ( + boolFalse vips.BoolParameter + intMinusOne vips.IntParameter +) + var _ = filetype.AddMatcher(filetype.NewType("svg", "image/svg+xml"), svgMatcher) func svgMatcher(buf []byte) bool { @@ -25,16 +34,15 @@ func svgMatcher(buf []byte) bool { } func GetFileContentType(filename string) string { - if strings.HasSuffix(filename, ".webp") { - return "image/webp" - } else if strings.HasSuffix(filename, ".avif") { - return "image/avif" - } else { - // raw image, need to use filetype to determine - buf, _ := os.ReadFile(filename) - kind, _ := filetype.Match(buf) - return kind.MIME.Value - } + // raw image, need to use filetype to determine + buf, _ := os.ReadFile(filename) + return GetContentType(buf) +} + +func GetContentType(buf []byte) string { + // raw image, need to use filetype to determine + kind, _ := filetype.Match(buf) + return kind.MIME.Value } func FileCount(dir string) int64 { @@ -54,7 +62,7 @@ func FileCount(dir string) int64 { func ImageExists(filename string) bool { info, err := os.Stat(filename) - if os.IsNotExist(err) { + if os.IsNotExist(err) || err != nil { return false } // if file size is less than 100 bytes, we assume it's invalid file @@ -83,25 +91,31 @@ func ImageExists(filename string) bool { return !info.IsDir() } -func CheckAllowedType(imgFilename string) bool { - for _, allowedType := range config.Config.AllowedTypes { - if allowedType == "*" { - return true - } - allowedType = "." + strings.ToLower(allowedType) - if strings.HasSuffix(strings.ToLower(imgFilename), allowedType) { - return true - } +func GetImageExtension(filename string) string { + return strings.TrimPrefix(strings.ToLower(path.Ext(filename)), ".") +} + +// CheckAllowedExtension checks if the image extension is in the user's allowed types +func CheckAllowedExtension(imgFilename string) bool { + if config.Config.AllowedTypes[0] == "*" { + return true } - return false + return slices.Contains(config.Config.AllowedTypes, GetImageExtension(imgFilename)) } -func GenOptimizedAbsPath(metadata config.MetaFile) (string, string) { +// CheckImageExtension checks if the image extension is in the WebP Server Go's default types +func CheckImageExtension(imgFilename string) bool { + return slices.Contains(config.DefaultAllowedTypes, GetImageExtension(imgFilename)) +} + +func GenOptimizedAbsPath(metadata config.MetaFile, subdir string) (string, string, string) { webpFilename := fmt.Sprintf("%s.webp", metadata.Id) avifFilename := fmt.Sprintf("%s.avif", metadata.Id) - webpAbsolutePath := path.Clean(path.Join(config.Config.ExhaustPath, webpFilename)) - avifAbsolutePath := path.Clean(path.Join(config.Config.ExhaustPath, avifFilename)) - return avifAbsolutePath, webpAbsolutePath + jxlFilename := fmt.Sprintf("%s.jxl", metadata.Id) + webpAbsolutePath := path.Clean(path.Join(config.Config.ExhaustPath, subdir, webpFilename)) + avifAbsolutePath := path.Clean(path.Join(config.Config.ExhaustPath, subdir, avifFilename)) + jxlAbsolutePath := path.Clean(path.Join(config.Config.ExhaustPath, subdir, jxlFilename)) + return avifAbsolutePath, webpAbsolutePath, jxlAbsolutePath } func GetCompressionRate(RawImagePath string, optimizedImg string) string { @@ -119,17 +133,23 @@ func GetCompressionRate(RawImagePath string, optimizedImg string) string { return fmt.Sprintf(`%.2f`, compressionRate) } -func GuessSupportedFormat(header *fasthttp.RequestHeader) []string { +func GuessSupportedFormat(header *fasthttp.RequestHeader) map[string]bool { var ( - supported = map[string]bool{ - "raw": true, - "webp": false, - "avif": false, - } - - ua = string(header.Peek("user-agent")) - accept = strings.ToLower(string(header.Peek("accept"))) + ua = string(header.Peek("user-agent")) + accept = strings.ToLower(string(header.Peek("accept"))) + supported = map[string]bool{} ) + // Initialize all supported formats to false + for _, item := range config.DefaultAllowedTypes { + supported[item] = false + } + // raw format(jpg,jpeg,png,gif) is always supported + supported["jpg"] = true + supported["jpeg"] = true + supported["png"] = true + supported["gif"] = true + supported["svg"] = true + supported["bmp"] = true if strings.Contains(accept, "image/webp") { supported["webp"] = true @@ -137,30 +157,49 @@ func GuessSupportedFormat(header *fasthttp.RequestHeader) []string { if strings.Contains(accept, "image/avif") { supported["avif"] = true } + if strings.Contains(accept, "image/jxl") { + supported["jxl"] = true + } + parsedUA := useragent.Parse(ua) - // chrome on iOS will not send valid image accept header - if strings.Contains(ua, "iPhone OS 14") || strings.Contains(ua, "CPU OS 14") || - strings.Contains(ua, "iPhone OS 15") || strings.Contains(ua, "CPU OS 15") || - strings.Contains(ua, "iPhone OS 16") || strings.Contains(ua, "CPU OS 16") || - strings.Contains(ua, "iPhone OS 17") || strings.Contains(ua, "CPU OS 17") || - strings.Contains(ua, "Android") || strings.Contains(ua, "Linux") { + if parsedUA.IsIOS() && parsedUA.VersionNo.Major >= 14 { supported["webp"] = true } - // iOS 16 supports AVIF - if strings.Contains(ua, "iPhone OS 16") || strings.Contains(ua, "CPU OS 16") || - strings.Contains(ua, "iPhone OS 17") || strings.Contains(ua, "CPU OS 17") { + if parsedUA.IsIOS() && parsedUA.VersionNo.Major >= 16 { supported["avif"] = true } - // save true value's key to slice - var accepted []string - for k, v := range supported { - if v { - accepted = append(accepted, k) - } + // Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.4 Safari/605.1.15 <- iPad + // Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.3.1 Safari/605.1.15 <- Mac + // Mozilla/5.0 (iPhone; CPU iPhone OS 17_4 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.4 Mobile/15E148 Safari/604.1 <- iPhone @ Safari + if parsedUA.IsIOS() && parsedUA.VersionNo.Major >= 17 { + supported["jxl"] = true + } + + if parsedUA.IsSafari() && parsedUA.VersionNo.Major >= 17 { + supported["heic"] = true + } + + // Firefox will not send correct accept header on url without image extension, we need to check user agent to see if `Firefox/133` version is supported + // https://caniuse.com/webp + if parsedUA.IsFirefox() && parsedUA.VersionNo.Major >= 133 { + supported["webp"] = true + } + + // https://caniuse.com/avif + if parsedUA.IsFirefox() && parsedUA.VersionNo.Major >= 93 { + supported["avif"] = true } - return accepted + + return supported +} + +func CopyFile(src, dst string) error { + // Read all content of src to data + data, _ := os.ReadFile(src) + // Write data to dst + return os.WriteFile(dst, data, 0644) } func FindSmallestFiles(files []string) string { diff --git a/helper/helper_test.go b/helper/helper_test.go index 1469918b3..ece445081 100644 --- a/helper/helper_test.go +++ b/helper/helper_test.go @@ -5,6 +5,7 @@ import ( "webp_server_go/config" "github.com/stretchr/testify/assert" + "github.com/valyala/fasthttp" ) func TestMain(m *testing.M) { @@ -12,7 +13,6 @@ func TestMain(m *testing.M) { config.LoadConfig() m.Run() config.ConfigPath = "config.json" - } func TestFileCount(t *testing.T) { @@ -37,12 +37,130 @@ func TestImageExists(t *testing.T) { }) } -func TestCheckAllowedType(t *testing.T) { +func TestCheckAllowedExtension(t *testing.T) { t.Run("not allowed type", func(t *testing.T) { - assert.False(t, CheckAllowedType("./helper_test.go")) + assert.False(t, CheckAllowedExtension("./helper_test.go")) }) t.Run("allowed type", func(t *testing.T) { - assert.True(t, CheckAllowedType("test.jpg")) + assert.True(t, CheckAllowedExtension("test.jpg")) }) } + +func TestGuessSupportedFormat(t *testing.T) { + tests := []struct { + name string + userAgent string + accept string + expected map[string]bool + }{ + { + name: "WebP/AVIF/JXL Supported", + userAgent: "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.4 Safari/605.1.15", // iPad + accept: "image/webp, image/avif", + expected: map[string]bool{ + "jpg": true, + "jpeg": true, + "png": true, + "gif": true, + "svg": true, + "bmp": true, + "webp": true, + "avif": true, + "jxl": false, + "nef": false, + "heic": true, + }, + }, + { + name: "WebP Supported", + userAgent: "iPhone OS 15", + accept: "image/webp, image/png", + expected: map[string]bool{ + "jpg": true, + "jpeg": true, + "png": true, + "gif": true, + "svg": true, + "bmp": true, + "webp": true, + "avif": false, + "jxl": false, + "nef": false, + "heic": false, + }, + }, + { + name: "WebP/AVIF Supported", + userAgent: "iPhone OS 16", + accept: "image/webp, image/png", + expected: map[string]bool{ + "jpg": true, + "jpeg": true, + "png": true, + "gif": true, + "svg": true, + "bmp": true, + "webp": true, + "avif": false, + "jxl": false, + "nef": false, + "heic": false, + }, + }, + { + name: "Both Supported", + userAgent: "iPhone OS 16", + accept: "image/webp, image/avif", + expected: map[string]bool{ + "jpg": true, + "jpeg": true, + "png": true, + "gif": true, + "svg": true, + "bmp": true, + "webp": true, + "avif": true, + "jxl": false, + "nef": false, + "heic": false, + }, + }, + { + name: "No Supported Formats", + userAgent: "Unknown OS", + accept: "image/jpeg, image/gif", + expected: map[string]bool{ + "jpg": true, + "jpeg": true, + "png": true, + "gif": true, + "svg": true, + "bmp": true, + "webp": false, + "avif": false, + "jxl": false, + "nef": false, + "heic": false, + }, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + header := &fasthttp.RequestHeader{} + header.Set("user-agent", test.userAgent) + header.Set("accept", test.accept) + + result := GuessSupportedFormat(header) + + if len(result) != len(test.expected) { + t.Errorf("Expected %v, but got %v", test.expected, result) + } + + for k, v := range test.expected { + assert.Equal(t, v, result[k]) + } + }) + } +} diff --git a/helper/metadata.go b/helper/metadata.go index 75f7599c2..4db0472b7 100644 --- a/helper/metadata.go +++ b/helper/metadata.go @@ -7,6 +7,8 @@ import ( "path" "webp_server_go/config" + "github.com/buckket/go-blurhash" + "github.com/davidbyttow/govips/v2/vips" log "github.com/sirupsen/logrus" ) @@ -18,37 +20,38 @@ func getId(p string) (string, string, string) { parsed, _ := url.Parse(p) width := parsed.Query().Get("width") height := parsed.Query().Get("height") - // santizedPath will be /webp_server.jpg?width=200\u0026height= in local mode when requesting /webp_server.jpg?width=200 + max_width := parsed.Query().Get("max_width") + max_height := parsed.Query().Get("max_height") + // santizedPath will be /webp_server.jpg?width=200\u0026height=\u0026max_width=\u0026max_height= in local mode when requesting /webp_server.jpg?width=200 // santizedPath will be https://docs.webp.sh/images/webp_server.jpg?width=400 in proxy mode when requesting /images/webp_server.jpg?width=400 with IMG_PATH = https://docs.webp.sh - santizedPath := parsed.Path + "?width=" + width + "&height=" + height + santizedPath := parsed.Path + "?width=" + width + "&height=" + height + "&max_width=" + max_width + "&max_height=" + max_height id = HashString(santizedPath) return id, path.Join(config.Config.ImgPath, parsed.Path), santizedPath } -func ReadMetadata(p, etag string) config.MetaFile { +func ReadMetadata(p, etag string, subdir string) config.MetaFile { // try to read metadata, if we can't read, create one var metadata config.MetaFile var id, _, _ = getId(p) - buf, err := os.ReadFile(path.Join(config.Metadata, id+".json")) - if err != nil { - log.Warnf("can't read metadata: %s", err) - WriteMetadata(p, etag) - return ReadMetadata(p, etag) - } - - err = json.Unmarshal(buf, &metadata) - if err != nil { - log.Warnf("unmarshal metadata error, possible corrupt file, re-building...: %s", err) - WriteMetadata(p, etag) - return ReadMetadata(p, etag) + if buf, err := os.ReadFile(path.Join(config.Config.MetadataPath, subdir, id+".json")); err != nil { + // First time reading metadata, create one + WriteMetadata(p, etag, subdir) + return ReadMetadata(p, etag, subdir) + } else { + err = json.Unmarshal(buf, &metadata) + if err != nil { + log.Warnf("unmarshal metadata error, possible corrupt file, re-building...: %s", err) + WriteMetadata(p, etag, subdir) + return ReadMetadata(p, etag, subdir) + } + return metadata } - return metadata } -func WriteMetadata(p, etag string) config.MetaFile { - _ = os.Mkdir(config.Metadata, 0755) +func WriteMetadata(p, etag string, subdir string) config.MetaFile { + _ = os.MkdirAll(path.Join(config.Config.MetadataPath, subdir), 0755) var id, filepath, sant = getId(p) @@ -56,7 +59,7 @@ func WriteMetadata(p, etag string) config.MetaFile { Id: id, } - if config.ProxyMode { + if etag != "" { data.Path = p data.Checksum = HashString(etag) } else { @@ -64,7 +67,109 @@ func WriteMetadata(p, etag string) config.MetaFile { data.Checksum = HashFile(filepath) } + imageMeta := getImageMeta(filepath) + data.ImageMeta = imageMeta + buf, _ := json.Marshal(data) - _ = os.WriteFile(path.Join(config.Metadata, data.Id+".json"), buf, 0644) + _ = os.WriteFile(path.Join(config.Config.MetadataPath, subdir, data.Id+".json"), buf, 0644) return data } + +func getImageMeta(filePath string) (metadata config.ImageMeta) { + boolFalse.Set(false) + intMinusOne.Set(-1) + img, err := vips.LoadImageFromFile(filePath, &vips.ImportParams{ + FailOnError: boolFalse, + NumPages: intMinusOne, + }) + if err != nil { + log.Warnf("Could not load %s: %s", filePath, err) + return metadata + } + defer img.Close() + var colorspace string + switch img.Interpretation() { + case vips.InterpretationSRGB: + colorspace = "sRGB" + case vips.InterpretationYXY: + colorspace = "YXY" + case vips.InterpretationFourier: + colorspace = "Fourier" + case vips.InterpretationGrey16: + colorspace = "Grey16" + case vips.InterpretationMatrix: + colorspace = "Matrix" + case vips.InterpretationScRGB: + colorspace = "scRGB" + case vips.InterpretationHSV: + colorspace = "HSV" + default: + colorspace = "Unknown" + } + // Get image size + height := img.Metadata().Height + width := img.Metadata().Width + numPages := img.Metadata().Pages + if numPages > 1 { + height = height / numPages + } + var imgFormat string + switch img.Format() { + case vips.ImageTypeJPEG: + imgFormat = "jpeg" + case vips.ImageTypePNG: + imgFormat = "png" + case vips.ImageTypeWEBP: + imgFormat = "webp" + case vips.ImageTypeAVIF: + imgFormat = "avif" + case vips.ImageTypeGIF: + imgFormat = "gif" + case vips.ImageTypeBMP: + imgFormat = "bmp" + default: + imgFormat = "unknown" + } + + imgBytes, err := img.ToBytes() + if err != nil { + log.Error("Error in img.ToBytes", err) + return + } + + metadata = config.ImageMeta{ + Width: width, + Height: height, + Format: imgFormat, + Colorspace: colorspace, + NumPages: numPages, + Size: len(imgBytes), + } + + // Get blurhash + _ = img.Thumbnail(32, 32, vips.InterestingAttention) + imageImage, err := img.ToImage(vips.NewDefaultExportParams()) + if err != nil { + log.Error("Error in img.ToImage", err) + return + } + + blurHash, err := blurhash.Encode(4, 3, imageImage) + if err != nil { + log.Error("Error in blurhash", err) + return + } + + metadata.Blurhash = blurHash + + return metadata +} + +func DeleteMetadata(p string, subdir string) { + var id, _, _ = getId(p) + metadataPath := path.Join(config.Config.MetadataPath, subdir, id+".json") + err := os.Remove(metadataPath) + if err != nil { + log.Warnln("failed to delete metadata", err) + } +} diff --git a/helper/metadata_test.go b/helper/metadata_test.go index 9f90227ae..1f0015b7b 100644 --- a/helper/metadata_test.go +++ b/helper/metadata_test.go @@ -32,9 +32,9 @@ func TestGetId(t *testing.T) { // Verify the return values parsed, _ := url.Parse(p) - expectedId := HashString(parsed.Path + "?width=400&height=500") + expectedId := HashString(parsed.Path + "?width=400&height=500&max_width=&max_height=") expectedPath := path.Join(config.Config.ImgPath, parsed.Path) - expectedSantizedPath := parsed.Path + "?width=400&height=500" + expectedSantizedPath := parsed.Path + "?width=400&height=500&max_width=&max_height=" if id != expectedId || jointPath != expectedPath || santizedPath != expectedSantizedPath { t.Errorf("Test case 2 failed: Expected (%s, %s, %s), but got (%s, %s, %s)", expectedId, expectedPath, expectedSantizedPath, id, jointPath, santizedPath) diff --git a/pics/blackbird.NEF b/pics/blackbird.NEF new file mode 100644 index 000000000..4f4e28deb Binary files /dev/null and b/pics/blackbird.NEF differ diff --git a/pics/config.json b/pics/config.json new file mode 100644 index 000000000..bb3e051b5 --- /dev/null +++ b/pics/config.json @@ -0,0 +1,16 @@ +{ + "HOST": "127.0.0.1", + "PORT": "3333", + "QUALITY": "80", + "IMG_PATH": "./pics", + "EXHAUST_PATH": "./exhaust", + "IMG_MAP": {}, + "ALLOWED_TYPES": ["jpg","png","jpeg","gif","bmp","svg","heic","nef"], + "CONVERT_TYPES": ["webp"], + "STRIP_METADATA": true, + "ENABLE_EXTRA_PARAMS": false, + "READ_BUFFER_SIZE": 4096, + "CONCURRENCY": 262144, + "DISABLE_KEEPALIVE": false, + "CACHE_TTL": 259200 +} \ No newline at end of file diff --git a/pics/kimono.avif b/pics/kimono.avif new file mode 100644 index 000000000..e91fe56e7 Binary files /dev/null and b/pics/kimono.avif differ diff --git a/pics/sample3.heic b/pics/sample3.heic new file mode 100644 index 000000000..dd76335bf Binary files /dev/null and b/pics/sample3.heic differ diff --git a/schedule/cache_clean.go b/schedule/cache_clean.go new file mode 100644 index 000000000..34740f3e0 --- /dev/null +++ b/schedule/cache_clean.go @@ -0,0 +1,115 @@ +package schedule + +import ( + "os" + "path/filepath" + "time" + "webp_server_go/config" + + log "github.com/sirupsen/logrus" +) + +func getDirSize(path string) (int64, error) { + // Check if path is a directory and exists + if _, err := os.Stat(path); os.IsNotExist(err) { + return 0, nil + } + var size int64 + err := filepath.Walk(path, func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + if !info.IsDir() { + size += info.Size() + } + return nil + }) + return size, err +} + +// Delete the oldest file in the given path +func clearDirForOldestFiles(path string) error { + oldestFile := "" + oldestModTime := time.Now() + + err := filepath.Walk(path, func(path string, info os.FileInfo, err error) error { + if err != nil { + log.Errorf("Error accessing path %s: %s\n", path, err.Error()) + return nil + } + + if !info.IsDir() && info.ModTime().Before(oldestModTime) { + oldestFile = path + oldestModTime = info.ModTime() + } + return nil + }) + + if err != nil { + log.Errorf("Error traversing directory: %s\n", err.Error()) + return err + } + + if oldestFile != "" { + err := os.Remove(oldestFile) + if err != nil { + log.Errorf("Error deleting file %s: %s\n", oldestFile, err.Error()) + return err + } + log.Infof("Deleted oldest file: %s\n", oldestFile) + } else { + log.Infoln("No files found in the directory.") + } + return nil +} + +// Clear cache, size is in bytes that needs to be cleared out +// Will delete oldest files first, then second oldest, etc. +// Until all files size are less than maxCacheSizeBytes +func clearCacheFiles(path string, maxCacheSizeBytes int64) error { + dirSize, err := getDirSize(path) + if err != nil { + log.Errorf("Error getting directory size: %s\n", err.Error()) + return err + } + + for dirSize > maxCacheSizeBytes { + err := clearDirForOldestFiles(path) + if err != nil { + log.Errorf("Error clearing directory: %s\n", err.Error()) + return err + } + dirSize, err = getDirSize(path) + if err != nil { + log.Errorf("Error getting directory size: %s\n", err.Error()) + return err + } + } + return nil +} + +func CleanCache() { + log.Info("MaxCacheSize is not 0, starting cache cleaning service") + ticker := time.NewTicker(1 * time.Minute) + defer ticker.Stop() + + for { + select { + case <-ticker.C: + // MB to bytes + maxCacheSizeBytes := int64(config.Config.MaxCacheSize) * 1024 * 1024 + err := clearCacheFiles(config.Config.RemoteRawPath, maxCacheSizeBytes) + if err != nil { + log.Warn("Failed to clear remote raw cache") + } + err = clearCacheFiles(config.Config.ExhaustPath, maxCacheSizeBytes) + if err != nil && err != os.ErrNotExist { + log.Warn("Failed to clear remote raw cache") + } + err = clearCacheFiles(config.Config.MetadataPath, maxCacheSizeBytes) + if err != nil && err != os.ErrNotExist { + log.Warn("Failed to clear remote raw cache") + } + } + } +} diff --git a/webp-server.go b/webp-server.go index 16a929369..987d5b44a 100644 --- a/webp-server.go +++ b/webp-server.go @@ -8,6 +8,7 @@ import ( "webp_server_go/config" "webp_server_go/encoder" "webp_server_go/handler" + schedule "webp_server_go/schedule" "github.com/gofiber/fiber/v2" "github.com/gofiber/fiber/v2/middleware/etag" @@ -16,11 +17,15 @@ import ( log "github.com/sirupsen/logrus" ) +// https://docs.gofiber.io/api/fiber var app = fiber.New(fiber.Config{ ServerHeader: "WebP Server Go", AppName: "WebP Server Go", DisableStartupMessage: true, ProxyHeader: "X-Real-IP", + ReadBufferSize: config.Config.ReadBufferSize, // per-connection buffer size for requests' reading. This also limits the maximum header size. Increase this buffer if your clients send multi-KB RequestURIs and/or multi-KB headers (for example, BIG cookies). + Concurrency: config.Config.Concurrency, // Maximum number of concurrent connections. + DisableKeepalive: config.Config.DisableKeepalive, // Disable keep-alive connections, the server will close incoming connections after sending the first response to the client }) func setupLogger() { @@ -35,63 +40,87 @@ func setupLogger() { }, } log.SetFormatter(formatter) - log.SetLevel(log.InfoLevel) - // fiber logger format - app.Use(logger.New(logger.Config{ - Format: config.FiberLogFormat, - TimeFormat: config.TimeDateFormat, - })) app.Use(recover.New(recover.Config{})) - log.Infoln("WebP Server Go ready.") + fmt.Println("Allowed file types as source:", config.Config.AllowedTypes) + fmt.Println("Convert to WebP Enabled:", config.Config.EnableWebP) + fmt.Println("Convert to AVIF Enabled:", config.Config.EnableAVIF) + fmt.Println("Convert to JXL Enabled:", config.Config.EnableJXL) } func init() { - // main init is the last one to be called - flag.Parse() - config.LoadConfig() - setupLogger() -} - -func main() { // Our banner banner := fmt.Sprintf(` -β–Œ β–Œ β–Œ β–›β–€β–– β–žβ–€β–– β–žβ–€β–– -β–Œβ––β–Œβ–žβ–€β––β–›β–€β––β–™β–„β–˜ β–šβ–„ β–žβ–€β––β–™β–€β––β–Œ β–Œβ–žβ–€β––β–™β–€β–– β–Œβ–„β––β–žβ–€β–– -β–™β–šβ–Œβ–›β–€ β–Œ β–Œβ–Œ β–– β–Œβ–›β–€ β–Œ ▐▐ β–›β–€ β–Œ β–Œ β–Œβ–Œ β–Œ -β–˜ β–˜β–β–€β–˜β–€β–€ β–˜ ▝▀ β–β–€β–˜β–˜ β–˜ β–β–€β–˜β–˜ ▝▀ ▝▀ + β–Œ β–Œ β–Œ β–›β–€β–– β–žβ–€β–– β–žβ–€β–– + β–Œβ––β–Œβ–žβ–€β––β–›β–€β––β–™β–„β–˜ β–šβ–„ β–žβ–€β––β–™β–€β––β–Œ β–Œβ–žβ–€β––β–™β–€β–– β–Œβ–„β––β–žβ–€β–– + β–™β–šβ–Œβ–›β–€ β–Œ β–Œβ–Œ β–– β–Œβ–›β–€ β–Œ ▐▐ β–›β–€ β–Œ β–Œ β–Œβ–Œ β–Œ + β–˜ β–˜β–β–€β–˜β–€β–€ β–˜ ▝▀ β–β–€β–˜β–˜ β–˜ β–β–€β–˜β–˜ ▝▀ ▝▀ + + WebP Server Go - v%s + Developed by WebP Server team. https://github.com/webp-sh`, config.Version) + // main init is the last one to be called + flag.Parse() + loglevel := config.Verbosity -WebP Server Go - v%s -Develop by WebP Server team. https://github.com/webp-sh`, config.Version) + // Only enable fiber logger if loglevel is greater than 0 + if loglevel > 0 { + // fiber logger format + app.Use(logger.New(logger.Config{ + Format: config.FiberLogFormat, + TimeFormat: config.TimeDateFormat, + })) + } + switch loglevel { + case 0: + log.SetLevel(log.PanicLevel) + case 1: + log.SetLevel(log.ErrorLevel) + case 2: + log.SetLevel(log.WarnLevel) + case 3: + log.SetLevel(log.InfoLevel) + case 4: + log.SetLevel(log.DebugLevel) + } // process cli params if config.DumpConfig { fmt.Println(config.SampleConfig) os.Exit(0) } - if config.DumpSystemd { - fmt.Println(config.SampleSystemd) - os.Exit(0) - } if config.ShowVersion { fmt.Printf("\n %c[1;32m%s%c[0m\n\n", 0x1B, banner+"", 0x1B) os.Exit(0) } + config.LoadConfig() + fmt.Printf("\n %c[1;32m%s%c[0m\n\n", 0x1B, banner, 0x1B) + setupLogger() +} +func main() { + if config.Config.MaxCacheSize != 0 { + go schedule.CleanCache() + } if config.Prefetch { go encoder.PrefetchImages() + } else if config.PrefetchForeground { + // Standalone prefetch, prefetch and exit + encoder.PrefetchImages() + os.Exit(0) } - app.Use(etag.New(etag.Config{ Weak: true, })) listenAddress := config.Config.Host + ":" + config.Config.Port + + app.Get("/healthz", handler.Healthz) app.Get("/*", handler.Convert) - fmt.Printf("\n %c[1;32m%s%c[0m\n\n", 0x1B, banner, 0x1B) fmt.Println("WebP Server Go is Running on http://" + listenAddress) - _ = app.Listen(listenAddress) - + bindErr := app.Listen(listenAddress) + if bindErr != nil { + log.Fatal("Error starting server: ", bindErr) + } }