From 0e97206907a5b8455279810cb2ec9d9475b1bbca Mon Sep 17 00:00:00 2001 From: Amir Pourmand Date: Fri, 22 May 2026 20:13:33 +0330 Subject: [PATCH] ci: optimize build time for Docker and CI workflows (#4263) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * ci: optimize build time for Docker and CI workflows - Dockerfile: slim production-stage to copy only runtime artifacts (.next, public, node_modules, package.json, next.config.mjs), dropping src/, src-tauri/, patches/, packages source, etc. - Dockerfile: add sharing=locked to pnpm store cache mount to prevent concurrent-build cache corruption - docker-image.yml: pin actions/checkout to SHA (consistent with other workflows) - docker-image.yml: switch Buildx cache from type=gha (10 GB shared limit, poor for multi-arch) to type=registry on GHCR (no size cap, already authenticated, correct per-platform caching) - pull-request.yml: use pnpm install --frozen-lockfile --prefer-offline - release.yml: use pnpm install --frozen-lockfile --prefer-offline - next.config.mjs: skip redundant eslint pass during next build (lint already runs as a dedicated CI step) * fix(docker): eliminate QEMU emulation, fix pnpm version mismatch, patch artifact write CVE (#4) * fix(docker): eliminate QEMU arm64 emulation and fix pnpm version mismatch - Fix pnpm version in Dockerfile: 10.29.3 → 11.1.1 (matches package.json) Prevents corepack from re-downloading pnpm 11 on every build - Replace single QEMU job with matrix build (ubuntu-latest for amd64, ubuntu-24.04-arm for arm64) — eliminates ~21 min QEMU emulation overhead - Use per-platform build cache tags (buildcache-linux-amd64 / buildcache-linux-arm64) to avoid cache thrashing between architectures - Add merge job that assembles multi-arch manifest from platform digests Agent-Logs-Url: https://github.com/pourmand1376/readest/sessions/89c298b4-8a58-4517-ac7f-2f26c86bbcd6 Co-authored-by: pourmand1376 <32064808+pourmand1376@users.noreply.github.com> * fix(ci): upgrade artifact actions to v7 to patch arbitrary file write vulnerability actions/download-artifact >= 4.0.0 < 4.1.3 allows arbitrary file write via artifact extraction. Pin both upload-artifact and download-artifact to v7 (SHA-pinned), consistent with the rest of the repo's workflows. Agent-Logs-Url: https://github.com/pourmand1376/readest/sessions/89c298b4-8a58-4517-ac7f-2f26c86bbcd6 Co-authored-by: pourmand1376 <32064808+pourmand1376@users.noreply.github.com> --------- Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com> Co-authored-by: pourmand1376 <32064808+pourmand1376@users.noreply.github.com> * chore(docker): tighten build context Exclude local env, build output, credential, and tooling state files from Docker build context to reduce registry cache exposure. --------- Co-authored-by: Copilot <198982749+Copilot@users.noreply.github.com> Co-authored-by: Huang Xin --- .dockerignore | 23 ++++- .github/workflows/docker-image.yml | 144 +++++++++++++++++++---------- .github/workflows/pull-request.yml | 6 +- .github/workflows/release.yml | 2 +- Dockerfile | 20 ++-- apps/readest-app/next.config.mjs | 5 + 6 files changed, 139 insertions(+), 61 deletions(-) diff --git a/.dockerignore b/.dockerignore index 0a6b6a89..66fa68a1 100644 --- a/.dockerignore +++ b/.dockerignore @@ -13,8 +13,30 @@ target # Build outputs .next **/.next +.open-next +**/.open-next out **/out +.vercel +**/.vercel + +# Local env files +docker/.env +.env*.local +**/.env*.local + +# Local credentials and tooling state +*.pem +certs +**/certs +.claude/settings.local.json +**/.claude/settings.local.json +.claude/worktrees +**/.claude/worktrees +.gstack +**/.gstack +.playwright-mcp +**/.playwright-mcp # IDE .idea @@ -28,4 +50,3 @@ Thumbs.db # Logs *.log npm-debug.log* - diff --git a/.github/workflows/docker-image.yml b/.github/workflows/docker-image.yml index 82ffded8..10c1ac49 100644 --- a/.github/workflows/docker-image.yml +++ b/.github/workflows/docker-image.yml @@ -18,18 +18,83 @@ permissions: packages: write jobs: - publish: - runs-on: ubuntu-latest + build: + runs-on: ${{ matrix.runner }} + strategy: + fail-fast: false + matrix: + include: + - platform: linux/amd64 + runner: ubuntu-latest + - platform: linux/arm64 + runner: ubuntu-24.04-arm env: BUILD_ARGS: | NEXT_PUBLIC_APP_PLATFORM=web steps: - - uses: actions/checkout@v4 + - name: Prepare platform pair + run: | + platform=${{ matrix.platform }} + echo "PLATFORM_PAIR=${platform//\//-}" >> "$GITHUB_ENV" + + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 with: submodules: recursive - - name: Set up QEMU - uses: docker/setup-qemu-action@v3 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Log in to GHCR + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Docker meta (for labels) + id: meta + uses: docker/metadata-action@v5 + with: + images: ghcr.io/${{ github.repository_owner }}/readest + + - name: Build and push by digest + id: build + uses: docker/build-push-action@v6 + with: + context: . + file: ./Dockerfile + target: production-stage + platforms: ${{ matrix.platform }} + labels: ${{ steps.meta.outputs.labels }} + build-args: ${{ env.BUILD_ARGS }} + cache-from: type=registry,ref=ghcr.io/${{ github.repository_owner }}/readest:buildcache-${{ env.PLATFORM_PAIR }} + cache-to: type=registry,ref=ghcr.io/${{ github.repository_owner }}/readest:buildcache-${{ env.PLATFORM_PAIR }},mode=max + outputs: type=image,name=ghcr.io/${{ github.repository_owner }}/readest,push-by-digest=true,name-canonical=true,push=true + + - name: Export digest + run: | + mkdir -p /tmp/digests + digest="${{ steps.build.outputs.digest }}" + touch "/tmp/digests/${digest#sha256:}" + + - name: Upload digest + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: digests-${{ env.PLATFORM_PAIR }} + path: /tmp/digests/* + if-no-files-found: error + retention-days: 1 + + merge: + needs: build + runs-on: ubuntu-latest + steps: + - name: Download digests + uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7 + with: + path: /tmp/digests + pattern: digests-* + merge-multiple: true - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 @@ -83,56 +148,35 @@ jobs: type=raw,value=${{ github.event.release.tag_name }},enable=${{ github.event_name == 'release' }} type=sha,prefix=sha- - - name: Build and push (GHCR only) - id: build-ghcr + - name: Create manifest list and push (GHCR only) if: steps.dockerhub.outputs.enabled != 'true' - uses: docker/build-push-action@v6 - with: - context: . - file: ./Dockerfile - target: production-stage - platforms: linux/amd64,linux/arm64 - push: true - tags: ${{ steps.meta-ghcr.outputs.tags }} - labels: ${{ steps.meta-ghcr.outputs.labels }} - build-args: ${{ env.BUILD_ARGS }} + working-directory: /tmp/digests + env: + DOCKER_METADATA_OUTPUT_JSON: ${{ steps.meta-ghcr.outputs.json }} + run: | + docker buildx imagetools create \ + $(jq -cr '.tags | map("-t " + .) | join(" ")' <<< "$DOCKER_METADATA_OUTPUT_JSON") \ + $(printf 'ghcr.io/${{ github.repository_owner }}/readest@sha256:%s ' *) - - name: Published image summary (GHCR only) - if: steps.dockerhub.outputs.enabled != 'true' + - name: Create manifest list and push (GHCR + Docker Hub) + if: steps.dockerhub.outputs.enabled == 'true' + working-directory: /tmp/digests + env: + DOCKER_METADATA_OUTPUT_JSON: ${{ steps.meta-all.outputs.json }} + run: | + docker buildx imagetools create \ + $(jq -cr '.tags | map("-t " + .) | join(" ")' <<< "$DOCKER_METADATA_OUTPUT_JSON") \ + $(printf 'ghcr.io/${{ github.repository_owner }}/readest@sha256:%s ' *) + + - name: Published image summary run: | echo "## Published Images" >> "$GITHUB_STEP_SUMMARY" echo "" >> "$GITHUB_STEP_SUMMARY" - echo "Registry: GHCR" >> "$GITHUB_STEP_SUMMARY" - echo "" >> "$GITHUB_STEP_SUMMARY" echo "Tags:" >> "$GITHUB_STEP_SUMMARY" echo '```' >> "$GITHUB_STEP_SUMMARY" - echo "${{ steps.meta-ghcr.outputs.tags }}" >> "$GITHUB_STEP_SUMMARY" + if [ "${{ steps.dockerhub.outputs.enabled }}" == "true" ]; then + echo "${{ steps.meta-all.outputs.tags }}" >> "$GITHUB_STEP_SUMMARY" + else + echo "${{ steps.meta-ghcr.outputs.tags }}" >> "$GITHUB_STEP_SUMMARY" + fi echo '```' >> "$GITHUB_STEP_SUMMARY" - echo "Digest: \`${{ steps.build-ghcr.outputs.digest }}\`" >> "$GITHUB_STEP_SUMMARY" - - - name: Build and push (GHCR + Docker Hub) - id: build-all - if: steps.dockerhub.outputs.enabled == 'true' - uses: docker/build-push-action@v6 - with: - context: . - file: ./Dockerfile - target: production-stage - platforms: linux/amd64,linux/arm64 - push: true - tags: ${{ steps.meta-all.outputs.tags }} - labels: ${{ steps.meta-all.outputs.labels }} - build-args: ${{ env.BUILD_ARGS }} - - - name: Published image summary (GHCR + Docker Hub) - if: steps.dockerhub.outputs.enabled == 'true' - run: | - echo "## Published Images" >> "$GITHUB_STEP_SUMMARY" - echo "" >> "$GITHUB_STEP_SUMMARY" - echo "Registries: GHCR, Docker Hub" >> "$GITHUB_STEP_SUMMARY" - echo "" >> "$GITHUB_STEP_SUMMARY" - echo "Tags:" >> "$GITHUB_STEP_SUMMARY" - echo '```' >> "$GITHUB_STEP_SUMMARY" - echo "${{ steps.meta-all.outputs.tags }}" >> "$GITHUB_STEP_SUMMARY" - echo '```' >> "$GITHUB_STEP_SUMMARY" - echo "Digest: \`${{ steps.build-all.outputs.digest }}\`" >> "$GITHUB_STEP_SUMMARY" diff --git a/.github/workflows/pull-request.yml b/.github/workflows/pull-request.yml index f94d5d69..f0ae34f2 100644 --- a/.github/workflows/pull-request.yml +++ b/.github/workflows/pull-request.yml @@ -72,7 +72,7 @@ jobs: - name: install Dependencies working-directory: apps/readest-app run: | - pnpm install && pnpm setup-vendors + pnpm install --frozen-lockfile --prefer-offline && pnpm setup-vendors - name: install LuaJIT (for koplugin lint) run: sudo apt-get update && sudo apt-get install -y luajit @@ -139,7 +139,7 @@ jobs: - name: install Dependencies working-directory: apps/readest-app run: | - pnpm install && pnpm setup-vendors + pnpm install --frozen-lockfile --prefer-offline && pnpm setup-vendors - name: cache apt packages uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5 @@ -216,7 +216,7 @@ jobs: - name: install Dependencies working-directory: apps/readest-app run: | - pnpm install && pnpm setup-vendors + pnpm install --frozen-lockfile --prefer-offline && pnpm setup-vendors - name: setup sccache uses: mozilla-actions/sccache-action@9e7fa8a12102821edf02ca5dbea1acd0f89a2696 # v0.0.10 diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 115f55fd..7ff1ee0a 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -186,7 +186,7 @@ jobs: run: sdkmanager "ndk;28.2.13676358" - name: install dependencies - run: pnpm install + run: pnpm install --frozen-lockfile --prefer-offline - name: copy pdfjs-dist and simplecc-dist to public directory run: pnpm --filter @readest/readest-app setup-vendors diff --git a/Dockerfile b/Dockerfile index d473a79b..9e5dcb2e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,20 +2,20 @@ FROM docker.io/library/node:24-slim@sha256:24dc26ef1e3c3690f27ebc4136c9c186c3133 ENV PNPM_HOME="/pnpm" ENV PATH="$PNPM_HOME:$PATH" RUN corepack enable -RUN corepack prepare pnpm@10.29.3 --activate +RUN corepack prepare pnpm@11.1.1 --activate WORKDIR /app COPY package.json pnpm-lock.yaml pnpm-workspace.yaml ./ COPY apps/readest-app/package.json ./apps/readest-app/ COPY patches/ ./patches/ COPY packages/ ./packages/ -RUN --mount=type=cache,id=pnpm,target=/pnpm/store pnpm install --frozen-lockfile +RUN --mount=type=cache,id=pnpm,sharing=locked,target=/pnpm/store pnpm install --frozen-lockfile RUN pnpm --filter @readest/readest-app setup-vendors FROM docker.io/library/node:24-slim@sha256:24dc26ef1e3c3690f27ebc4136c9c186c3133b25563ae4d7f0692e4d1fe5db0e AS development-stage ENV PNPM_HOME="/pnpm" ENV PATH="$PNPM_HOME:$PATH" RUN corepack enable -RUN corepack prepare pnpm@10.29.3 --activate +RUN corepack prepare pnpm@11.1.1 --activate WORKDIR /app COPY --from=dependencies /app /app COPY . . @@ -27,7 +27,7 @@ FROM docker.io/library/node:24-slim@sha256:24dc26ef1e3c3690f27ebc4136c9c186c3133 ENV PNPM_HOME="/pnpm" ENV PATH="$PNPM_HOME:$PATH" RUN corepack enable -RUN corepack prepare pnpm@10.29.3 --activate +RUN corepack prepare pnpm@11.1.1 --activate WORKDIR /app ARG NEXT_PUBLIC_SUPABASE_URL ARG NEXT_PUBLIC_SUPABASE_ANON_KEY @@ -48,9 +48,17 @@ FROM docker.io/library/node:24-slim@sha256:24dc26ef1e3c3690f27ebc4136c9c186c3133 ENV PNPM_HOME="/pnpm" ENV PATH="$PNPM_HOME:$PATH" RUN corepack enable -RUN corepack prepare pnpm@10.29.3 --activate +RUN corepack prepare pnpm@11.1.1 --activate WORKDIR /app -COPY --from=build /app /app +# Only copy what next start needs — omit source, Rust, tests, patches, etc. +COPY --from=build /app/package.json /app/package.json +COPY --from=build /app/pnpm-workspace.yaml /app/pnpm-workspace.yaml +COPY --from=build /app/node_modules /app/node_modules +COPY --from=build /app/apps/readest-app/package.json /app/apps/readest-app/package.json +COPY --from=build /app/apps/readest-app/next.config.mjs /app/apps/readest-app/next.config.mjs +COPY --from=build /app/apps/readest-app/node_modules /app/apps/readest-app/node_modules +COPY --from=build /app/apps/readest-app/.next /app/apps/readest-app/.next +COPY --from=build /app/apps/readest-app/public /app/apps/readest-app/public WORKDIR /app/apps/readest-app ENTRYPOINT ["pnpm", "start-web", "-H", "0.0.0.0"] EXPOSE 3000 diff --git a/apps/readest-app/next.config.mjs b/apps/readest-app/next.config.mjs index cad5ea63..522d469a 100644 --- a/apps/readest-app/next.config.mjs +++ b/apps/readest-app/next.config.mjs @@ -20,6 +20,11 @@ const nextConfig = { // Ensure Next.js uses SSG instead of SSR // https://nextjs.org/docs/pages/building-your-application/deploying/static-exports output: exportOutput ? 'export' : undefined, + // Linting runs as a dedicated CI step (`pnpm lint`), so skipping it during + // `next build` avoids duplicate work and shortens build time. + eslint: { + ignoreDuringBuilds: true, + }, pageExtensions: exportOutput ? ['jsx', 'tsx'] : ['js', 'jsx', 'ts', 'tsx'], // Note: This feature is required to use the Next.js Image component in SSG mode. // See https://nextjs.org/docs/messages/export-image-api for different workarounds.