improve build readability and speed (by parallelization) (#117)

.github/workflows/build.yaml

@@ -2,94 +2,223 @@ name: build from source
 
 on:
   push:
-    branches:
-      - master
-      - dev
     paths:
       - .github/workflows/build.yaml
       - build/*
-      - deb/*
+      - VERSION
   pull_request:
     paths:
       - .github/workflows/build.yaml
       - build/*
-      - deb/*
   workflow_dispatch:
 
 env:
-  DOCKER_REPO: shenxn/protonmail-bridge
+  GHCR_REPO: shenxn/protonmail-bridge-docker
-  DOCKER_REPO_DEV: ghcr.io/shenxn/protonmail-bridge-dev
+  DOCKERHUB_REPO: shenxn/protonmail-bridge
+  DOCKER_REPO_DEV: ghcr.io/shenxn/protonmail-bridge
   PLATFORMS: linux/amd64,linux/arm64/v8,linux/arm/v7,linux/riscv64
 
 jobs:
-  build:
+  test:
     runs-on: ubuntu-latest
-    services:
+    if: github.ref != 'refs/heads/master'
-      registry:
-        image: registry:2
-        ports:
-          - 5000:5000
     steps:
       - name: Checkout
         uses: actions/checkout@master
+
       - name: Set version
         id: version
-        run: echo "::set-output name=version::`cat build/VERSION`"
+        run: echo "version=`cat VERSION`" >> $GITHUB_ENV
-      - name: Set repo
+
-        id: repo
-        run: if [[ $GITHUB_REF == "refs/heads/master" ]]; then echo "::set-output name=repo::${DOCKER_REPO}"; else echo "::set-output name=repo::${DOCKER_REPO_DEV}"; fi
       - name: Docker meta
-        id: docker_meta
+        id: meta
-        uses: crazy-max/ghaction-docker-meta@v1
+        uses: docker/metadata-action@v5
         with:
-          images: ${{ steps.repo.outputs.repo }}
+          images: |
-      - name: Set up QEMU
+            ${{ env.DOCKER_REPO_DEV }}
-        uses: docker/setup-qemu-action@v1
+
+      - name: Login to GHCR
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.repository_owner }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v1
+        uses: docker/setup-buildx-action@v3
         with:
           driver-opts: network=host
-      - name: Build image without push to registry
+
-        uses: docker/build-push-action@v2
+      - name: Build and push by digest
+        id: build
+        uses: docker/build-push-action@v6
         with:
+          labels: ${{ steps.meta.outputs.labels }}
+          outputs: type=image,"name=${{ env.DOCKER_REPO_DEV }}",push-by-digest=false,name-canonical=true,push=true
           context: ./build
           file: ./build/Dockerfile
-          platforms: ${{ env.PLATFORMS }}
+          tags: "${{ env.DOCKER_REPO_DEV }}:dev-${{ github.ref_name }}"
-          push: true
+          build-args: |
-          tags: localhost:5000/protonmail-bridge:latest
+            version=${{ env.version }}
-      - name: Scan image
+
-        id: scan
+      - name: Run Trivy vulnerability scan
-        uses: anchore/scan-action@v2
+        uses: aquasecurity/trivy-action@0.30.0
         with:
-          image: localhost:5000/protonmail-bridge:latest
+          image-ref: "${{ env.DOCKER_REPO_DEV }}:dev-${{ github.ref_name }}"
-          fail-build: true
+          format: 'sarif'
-          severity-cutoff: critical
+          exit-code: 0
-          acs-report-enable: true
+          severity: 'CRITICAL,HIGH'
-      - name: Upload Anchore scan SARIF report
+          output: 'trivy-results.sarif'
-        uses: github/codeql-action/upload-sarif@v1
+
+      - name: Upload Trivy scan SARIF report
+        uses: github/codeql-action/upload-sarif@v3
         with:
-          sarif_file: ${{ steps.scan.outputs.sarif }}
+          sarif_file: 'trivy-results.sarif'
+
+  build:
+    runs-on: ubuntu-latest
+    if: github.event_name == 'push' && github.ref == 'refs/heads/master'
+    strategy:
+      fail-fast: false
+      matrix:
+        platform:
+          - linux/amd64
+          - linux/arm64/v8
+          - linux/arm/v7
+          - linux/riscv64
+    steps:
+      - name: Checkout
+        uses: actions/checkout@master
+
+      - name: Prepare
+        run: |
+          platform=${{ matrix.platform }}
+          echo "PLATFORM_PAIR=${platform//\//-}" >> $GITHUB_ENV
+
+      - name: Set version
+        id: version
+        run: echo "version=`cat VERSION`" >> $GITHUB_ENV
+
+      - name: Docker meta
+        id: meta
+        uses: docker/metadata-action@v5
+        with:
+          images: |
+            ${{ env.DOCKERHUB_REPO }}
+            ${{ env.GHCR_REPO }}
+
       - name: Login to Docker Hub
-        uses: docker/login-action@v1
+        uses: docker/login-action@v3
-        if: ${{ github.event_name != 'pull_request' && github.ref == 'refs/heads/master' }}
+        with:
+          username: ${{ vars.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+
+      - name: Login to GHCR
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.repository_owner }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v3
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Build and push by digest
+        id: build
+        uses: docker/build-push-action@v6
+        with:
+          platforms: ${{ matrix.platform }}
+          labels: ${{ steps.meta.outputs.labels }}
+          outputs: type=image,"name=name=${{ env.DOCKERHUB_REPO }},${{ env.GHCR_REPO }}",push-by-digest=true,name-canonical=true,push=true
+          context: ./build
+          file: ./build/Dockerfile
+          tags: "${{ env.version }}-build"
+          provenance: true
+          sbom: true
+          build-args: |
+            version=${{ env.version }}
+
+      - name: Export digest
+        run: |
+          mkdir -p ${{ runner.temp }}/digests
+          digest="${{ steps.build.outputs.digest }}"
+          touch "${{ runner.temp }}/digests/${digest#sha256:}"
+
+      - name: Upload digest
+        uses: actions/upload-artifact@v4
+        with:
+          name: digests-${{ env.PLATFORM_PAIR }}
+          path: ${{ runner.temp }}/digests/*
+          if-no-files-found: error
+          retention-days: 1
+
+
+  merge:
+    runs-on: ubuntu-latest
+    needs:
+      - build
+    steps:
+      - name: Download digests
+        uses: actions/download-artifact@v4
+        with:
+          path: ${{ runner.temp }}/digests
+          pattern: digests-*
+          merge-multiple: true
+
+      - name: Login to Docker Hub
+        uses: docker/login-action@v3
         with:
           username: ${{ secrets.REGISTRY_USERNAME }}
           password: ${{ secrets.REGISTRY_PASSWORD }}
-      - name: Login to GitHub Container Registry
+
-        uses: docker/login-action@v1
+      - name: Login to GHCR
-        if: ${{ github.event_name != 'pull_request' && github.ref == 'refs/heads/dev' }}
+        uses: docker/login-action@v3
         with:
           registry: ghcr.io
           username: ${{ github.repository_owner }}
           password: ${{ secrets.CR_PAT }}
-      - name: Push image
+
-        uses: docker/build-push-action@v2
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
         with:
-          context: ./build
+          driver-opts: network=host
-          file: ./build/Dockerfile
+
-          platforms: ${{ env.PLATFORMS }}
+      - name: Docker meta
+        id: meta
+        uses: docker/metadata-action@v5
+        with:
+          images: |
+            ${{ env.DOCKERHUB_REPO }}
+            ${{ env.GHCR_REPO }}
           tags: |
-            ${{ steps.repo.outputs.repo }}:build
+            type=raw,enable=true,value=${{ env.version }}-build
-            ${{ steps.repo.outputs.repo }}:${{ steps.version.outputs.version }}-build
+            type=raw,enable=true,suffix=,value=build
-          labels: ${{ steps.docker_meta.outputs.labels }}
+
-          push: ${{ github.event_name != 'pull_request' }}
+      - name: Create manifest list and push
+        working-directory: ${{ runner.temp }}/digests
+        run: |
+          docker buildx imagetools create $(jq -cr '.tags | map("-t " + .) | join(" ")' <<< "$DOCKER_METADATA_OUTPUT_JSON") \
+            $(printf '${{ env.DOCKERHUB_REPO }}@sha256:%s ' *)
+          docker buildx imagetools create $(jq -cr '.tags | map("-t " + .) | join(" ")' <<< "$DOCKER_METADATA_OUTPUT_JSON") \
+            $(printf '${{ env.GHCR_REPO }}@sha256:%s ' *)
+
+      - name: Run Trivy vulnerability scan
+        uses: aquasecurity/trivy-action@0.30.0
+        with:
+          image-ref: "${{ env.DOCKERHUB_REPO }}:${{ env.version }}-build"
+          format: 'sarif'
+          exit-code: 0
+          severity: 'CRITICAL,HIGH'
+          output: 'trivy-results.sarif'
+      - name: Upload Trivy scan SARIF report
+        uses: github/codeql-action/upload-sarif@v3
+        with:
+          sarif_file: 'trivy-results.sarif'
+
+      - name: Inspect image
+        run: |
+          docker buildx imagetools inspect ${{ env.DOCKERHUB_REPO }}:${{ steps.meta.outputs.version }}
+          docker buildx imagetools inspect ${{ env.GHCR_REPO }}:${{ steps.meta.outputs.version }}

.github/workflows/deb.yaml

@@ -2,16 +2,15 @@ name: pack from deb
 
 on:
   push:
-    branches:
-      - master
-      - dev
     paths:
       - .github/workflows/deb.yaml
       - deb/*
+      - VERSION
   pull_request:
     paths:
       - .github/workflows/deb.yaml
       - deb/*
+  workflow_dispatch:
 
 env:
   DOCKER_REPO: shenxn/protonmail-bridge
@@ -25,7 +24,7 @@ jobs:
         uses: actions/checkout@master
       - name: Set version
         id: version
-        run: echo "::set-output name=version::`cat deb/VERSION`"
+        run: echo "version=`cat VERSION`" >> $GITHUB_ENV
       - name: Set repo
         id: repo
         run: if [[ $GITHUB_REF == "refs/heads/master" ]]; then echo "::set-output name=repo::${DOCKER_REPO}"; else echo "::set-output name=repo::${DOCKER_REPO_DEV}"; fi
@@ -52,18 +51,18 @@ jobs:
           severity-cutoff: critical
           acs-report-enable: true
       - name: Upload Anchore scan SARIF report
-        uses: github/codeql-action/upload-sarif@v1
+        uses: github/codeql-action/upload-sarif@v3
         with:
           sarif_file: ${{ steps.scan.outputs.sarif }}
       - name: Login to DockerHub
         uses: docker/login-action@v1
-        if: ${{ github.event_name != 'pull_request' && github.ref == 'refs/heads/master' }}
+        if: ${{ github.event_name != 'pull_request' }}
         with:
           username: ${{ secrets.REGISTRY_USERNAME }}
           password: ${{ secrets.REGISTRY_PASSWORD }}
       - name: Login to GitHub Container Registry
         uses: docker/login-action@v1
-        if: ${{ github.event_name != 'pull_request' && github.ref == 'refs/heads/dev' }}
+        if: ${{ github.event_name != 'pull_request' }}
         with:
           registry: ghcr.io
           username: ${{ github.repository_owner }}
@@ -75,6 +74,6 @@ jobs:
           file: ./deb/Dockerfile
           tags: |
             ${{ steps.repo.outputs.repo }}:latest
-            ${{ steps.repo.outputs.repo }}:${{ steps.version.outputs.version }}
+            ${{ steps.repo.outputs.repo }}:${{ env.version }}
           labels: ${{ steps.docker_meta.outputs.labels }}
           push: ${{ github.event_name != 'pull_request' }}

.github/workflows/update-check.yaml

@@ -2,9 +2,6 @@ name: update check
 
 on:
   push:
-    branches:
-      - master
-      - dev
     paths:
       - .github/workflows/update-check.yaml
       - update-check.py

VERSION

@@ -0,0 +1 @@
+v3.19.0

build/.dockerignore

@@ -1,8 +0,0 @@
-*
-
-!.dockerignore
-!VERSION
-!entrypoint.sh
-!gpgparams
-!Dockerfile
-!build.sh

build/Dockerfile

@@ -1,13 +1,15 @@
 # The build image could be golang, but it currently does not support riscv64. Only debian:sid does, at the time of writing.
 FROM debian:sid-slim AS build
 
+ARG version
+
 # Install dependencies
-RUN apt-get update && apt-get install -y git golang build-essential libsecret-1-dev
+RUN apt-get update && apt-get install -y golang build-essential libsecret-1-dev
 
 # Build
+ADD https://github.com/ProtonMail/proton-bridge.git#${version} /build/
 WORKDIR /build/
-COPY build.sh VERSION /build/
+RUN make build-nogui vault-editor
-RUN bash build.sh
 
 FROM debian:sid-slim
 LABEL maintainer="Simon Felding <sife@adm.ku.dk>"
@@ -24,8 +26,8 @@ RUN apt-get update \
 COPY gpgparams entrypoint.sh /protonmail/
 
 # Copy protonmail
-COPY --from=build /build/proton-bridge/bridge /protonmail/
+COPY --from=build /build/bridge /protonmail/
-COPY --from=build /build/proton-bridge/proton-bridge /protonmail/
+COPY --from=build /build/proton-bridge /protonmail/
-COPY --from=build /build/proton-bridge/vault-editor /protonmail/
+COPY --from=build /build/vault-editor /protonmail/
 
 ENTRYPOINT ["bash", "/protonmail/entrypoint.sh"]

build/VERSION

@@ -1 +0,0 @@
-3.19.0

build/build.sh

@@ -1,13 +0,0 @@
-#!/bin/bash
-
-set -ex
-
-VERSION=`cat VERSION`
-
-# Clone new code
-git clone https://github.com/ProtonMail/proton-bridge.git
-cd proton-bridge
-git checkout v$VERSION
-
-# Build
-make build-nogui vault-editor

deb/.dockerignore

@@ -1,8 +0,0 @@
-*
-
-!.dockerignore
-!VERSION
-!entrypoint.sh
-!install.sh
-!gpgparams
-!Dockerfile

deb/Dockerfile

@@ -1,5 +1,16 @@
+### The Deb install is just a repack of the official ProtonMail Bridge deb package with less dependencies.
+### I recommend you don't use this. It's here for legacy reasons.
+
+FROM debian:sid-slim AS build
+
+COPY install.sh PACKAGE /
+RUN apt-get update && apt-get install -y wget binutils
+
+# Repack deb (removes unnecessary dependencies and produces /protonmail.deb)
+RUN bash /install.sh
+
 FROM debian:sid-slim
-LABEL maintainer="Xiaonan Shen <s@sxn.dev>"
+LABEL maintainer="Simon Felding <sife@adm.ku.dk>"
 
 EXPOSE 25/tcp
 EXPOSE 143/tcp
@@ -7,16 +18,11 @@ EXPOSE 143/tcp
 WORKDIR /protonmail
 
 # Copy bash scripts
-COPY gpgparams install.sh entrypoint.sh VERSION /protonmail/
+COPY gpgparams entrypoint.sh PACKAGE /protonmail/
+COPY --from=build /protonmail.deb /tmp/protonmail.deb
 
 RUN apt-get update \
-    && apt-get install -y --no-install-recommends socat pass procps libsecret-1-0 ca-certificates \
+    && apt-get install -y --no-install-recommends /tmp/protonmail.deb socat pass libsecret-1-0 ca-certificates procps \
     && rm -rf /var/lib/apt/lists/*
 
-# Copy bash scripts
+CMD ["bash", "/protonmail/entrypoint.sh"]
-COPY gpgparams entrypoint.sh /protonmail/
-
-# Install dependencies and protonmail bridge
-RUN bash install.sh
-
-ENTRYPOINT ["bash", "/protonmail/entrypoint.sh"]

deb/PACKAGE

@@ -0,0 +1 @@
+https://github.com/ProtonMail/proton-bridge/releases/download/v3.19.0/protonmail-bridge_3.19.0-1_amd64.deb

deb/VERSION

@@ -1 +0,0 @@
-3.19.0-1

deb/install.sh

@@ -1,36 +1,16 @@
 #!/bin/bash
-
 set -ex
 
-VERSION=`cat VERSION`
-DEB_FILE=protonmail-bridge_${VERSION}_amd64.deb
-
-# Install dependents
-apt-get update
-apt-get install -y --no-install-recommends socat pass ca-certificates
-
-# Build time dependencies
-apt-get install -y wget binutils xz-utils
-
 # Repack deb (remove unnecessary dependencies)
 mkdir deb
+wget -i /PACKAGE -O /deb/protonmail.deb
 cd deb
-wget -q https://protonmail.com/download/bridge/${DEB_FILE}
+ar x -v protonmail.deb
-ar x -v ${DEB_FILE}
 mkdir control
 tar zxvf control.tar.gz -C control
 sed -i "s/^Depends: .*$/Depends: libgl1, libc6, libsecret-1-0, libstdc++6, libgcc1/" control/control
 cd control
 tar zcvf ../control.tar.gz .
 cd ../
-ar rcs -v ${DEB_FILE} debian-binary control.tar.gz data.tar.gz
-cd ../
 
-# Install protonmail bridge
+ar rcs -v /protonmail.deb debian-binary control.tar.gz data.tar.gz
-apt-get install -y --no-install-recommends ./deb/${DEB_FILE}
-
-# Cleanup
-apt-get purge -y wget binutils xz-utils
-apt-get autoremove -y
-rm -rf /var/lib/apt/lists/*
-rm -rf deb

update-check.py

@@ -1,64 +1,37 @@
-import sys
+import requests, os, sys
-import os
-import requests
-import json
-import re
 
+def git(command):
+  return os.system(f"git {command}")
+
+
+release = requests.get("https://api.github.com/repos/protonmail/proton-bridge/releases/latest").json()
+version = release['tag_name']
+deb = [asset for asset in release ['assets'] if asset['name'].endswith('.deb')][0]['browser_download_url']
+
+print(f"Latest release is: {version}")
+
+with open("VERSION", 'w') as f:
+  f.write(version)
+
+with open("deb/PACKAGE", 'w') as f:
+  f.write(deb)
+
+git("config --local user.name 'GitHub Actions'")
+git("config --local user.email 'actions@github.com'")
+
+git("add -A")
+
+if git("diff --cached --quiet") == 0: # Returns 0 if there are no changes
+  print("Version didn't change")
+  exit(0)
+
+git(f"commit -m 'Bump version to {version}'")
 is_pull_request = sys.argv[1] == "true"
-print(f"is_pull_request={is_pull_request}")
 
-
-def check_version(directory, new_version):
-    print(f"Checking version for {directory}")
-
-    if not new_version:
-        print("Failed to get new version. Exiting.")
-        exit(1)
-
-    with open(f"{directory}/VERSION", "r") as f:
-        old_version = f.read().rstrip()
-    
-    print(f"Up-to-date version {new_version}")
-    print(f"Current version: {old_version}")
-
-    if old_version != new_version:
-        print(f"New release found: {new_version}")
-
-        # bump up to new release
-        with open(f"{directory}/VERSION", "w") as f:
-            f.write(new_version)
-        # commit
-        result = os.system(f"git config --local user.email 'actions@github.com' \
-            && git config --local user.name 'GitHub Actions' \
-            && git add {directory}/VERSION \
-            && git commit -m 'Bump {directory} version to {new_version}'")
-        if result != 0:
-            print("Failed to commit the bump. Exiting")
-            exit(1)
 if is_pull_request:
-            print("Action triggered by pull request. Do not push.")
+  print("This is a pull request, skipping push step.")
-        else:
+  exit(0)
-            result = os.system("git push")
+
-            if result != 0:
+if git("push") != 0:
-                print("Failed to push. Exiting")
+  print("Git push failed!")
   exit(1)
-    else:
-        print(f"Already newest version {old_version}")
-
-
-# check deb version
-response = requests.get("https://protonmail.com/download/current_version_linux.json")
-content = json.loads(response.content)
-version = re.match(".*_([0-9.-]+)_amd64\.deb", content["DebFile"]).group(1)
-check_version("deb", version)
-
-
-# check build version
-response = requests.get(
-    "https://api.github.com/repos/ProtonMail/proton-bridge/tags",
-    headers={"Accept": "application/vnd.github.v3+json"},
-    )
-tags = json.loads(response.content)
-version_re = re.compile("v\d+\.\d+\.\d+")
-releases = [tag["name"][1:] for tag in tags if version_re.match(tag["name"])]
-check_version("build", releases[0])