From 57384ceca30234336b1ab4ada00a4ab1212bd78c Mon Sep 17 00:00:00 2001 From: Kristóf Marussy Date: Sun, 2 Jun 2024 14:49:14 +0200 Subject: build: publish CLI as Docker container --- docker/Dockerfile | 71 -------------------------------------------------- docker/Dockerfile.base | 58 +++++++++++++++++++++++++++++++++++++++++ docker/Dockerfile.cli | 26 ++++++++++++++++++ docker/Dockerfile.web | 27 +++++++++++++++++++ docker/build.sh | 62 +++++++++++++++++++++++++++---------------- docker/docker-bake.hcl | 39 +++++++++++++++++++++++++++ 6 files changed, 189 insertions(+), 94 deletions(-) delete mode 100644 docker/Dockerfile create mode 100644 docker/Dockerfile.base create mode 100644 docker/Dockerfile.cli create mode 100644 docker/Dockerfile.web create mode 100644 docker/docker-bake.hcl diff --git a/docker/Dockerfile b/docker/Dockerfile deleted file mode 100644 index 6cb4d71c..00000000 --- a/docker/Dockerfile +++ /dev/null @@ -1,71 +0,0 @@ -# SPDX-FileCopyrightText: 2023 The Refinery Authors -# -# SPDX-License-Identifier: EPL-2.0 - -FROM public.ecr.aws/amazoncorretto/amazoncorretto:21-al2023-jdk AS jdk - -FROM --platform=$BUILDPLATFORM docker.io/eclipse-temurin:21-jdk-jammy AS jlink-base -# Use the Ubuntu Jammy based Temurin image for JLink, because it already -# contains the require objcopy tool for stripping debug symbols: -# https://github.com/docker-library/openjdk/issues/351 -# We'll have to cross-jlink the JDK: https://stackoverflow.com/a/47611708 -COPY --link --from=jdk /usr/lib/jvm/java-21-amazon-corretto /crossjdk - -FROM --platform=$BUILDPLATFORM jlink-base AS jlink-amd64-on-amd64 - -FROM --platform=$BUILDPLATFORM jlink-base AS jlink-arm64-on-arm64 - -FROM --platform=$BUILDPLATFORM jlink-base AS jlink-arm64-on-amd64 -# When cross-building the Docker image, we have the wrong toolchain for the -# target architecture. We forcibly override the target for the objcopy tool. -# The cross toolchain is not available in Amazon Linux 2023. -RUN apt-get update && \ - apt-get -y install binutils-aarch64-linux-gnu && \ - ln -sf /usr/bin/aarch64-linux-gnu-objcopy /usr/bin/objcopy && \ - rm -rf /var/lib/apt/lists/* - -FROM --platform=$BUILDPLATFORM jlink-base AS jlink-amd64-on-arm64 -RUN apt-get update && \ - apt-get -y install binutils-x86-64-linux-gnu && \ - ln -sf /usr/bin/x86_64-linux-gnu-objcopy /usr/bin/objcopy && \ - rm -rf /var/lib/apt/lists/* - -FROM --platform=$BUILDPLATFORM jlink-$TARGETARCH-on-$BUILDARCH as jlink -RUN jlink --no-header-files --no-man-pages --compress=2 \ - --module-path=/crossjdk/jmods --strip-debug --add-modules \ - java.base,java.logging,java.xml,jdk.zipfs \ - --output /jlink - -FROM public.ecr.aws/amazonlinux/amazonlinux:2023-minimal AS base -# The launcher script generated by Gradle uses xargs to parse the argument list. -RUN dnf install -y findutils && \ - dnf clean all -# The first layer contains the slimmed down JRE. -COPY --link --from=jlink /jlink /usr/lib/java -ENV JAVA_HOME="/usr/lib/java" PATH="/usr/lib/java/bin:${PATH}" -# Layer with platform-independent dependencies, slow changing. -ADD --link lib /app/lib - -FROM base AS refinery-amd64 -# Layer with platform-dependent dependencies, slow changing. -ADD --link lib_amd64 /app/lib -# Layer with platform-dependent startup script containing references to all -# dependency version. -ADD --link app_amd64_bin /app/bin - -FROM base AS refinery-arm64 -# Layer with platform-dependent dependencies, slow changing. -ADD --link lib_arm64 /app/lib -# Layer with platform-dependent startup script containing references to all -# dependency version. -ADD --link app_arm64_bin /app/bin - -FROM refinery-$TARGETARCH -# Layer with platform-independent application jars. -ADD --link app_lib /app/lib -# Common settings added on top. -ENV REFINERY_LISTEN_HOST=0.0.0.0 REFINERY_LISTEN_PORT=8888 -EXPOSE 8888 -USER 1000 -WORKDIR /app -ENTRYPOINT /app/bin/refinery-language-web diff --git a/docker/Dockerfile.base b/docker/Dockerfile.base new file mode 100644 index 00000000..0668ce78 --- /dev/null +++ b/docker/Dockerfile.base @@ -0,0 +1,58 @@ +# SPDX-FileCopyrightText: 2023-2024 The Refinery Authors +# +# SPDX-License-Identifier: EPL-2.0 + +FROM public.ecr.aws/amazoncorretto/amazoncorretto:21-al2023-jdk AS jdk + +FROM --platform=$BUILDPLATFORM docker.io/eclipse-temurin:21-jdk-jammy AS jlink-base +# Use the Ubuntu Jammy based Temurin image for JLink, because it already +# contains the require objcopy tool for stripping debug symbols: +# https://github.com/docker-library/openjdk/issues/351 +# We'll have to cross-jlink the JDK: https://stackoverflow.com/a/47611708 +COPY --link --from=jdk /usr/lib/jvm/java-21-amazon-corretto /crossjdk + +FROM --platform=$BUILDPLATFORM jlink-base AS jlink-amd64-on-amd64 + +FROM --platform=$BUILDPLATFORM jlink-base AS jlink-arm64-on-arm64 + +FROM --platform=$BUILDPLATFORM jlink-base AS jlink-arm64-on-amd64 +# When cross-building the Docker image, we have the wrong toolchain for the +# target architecture. We forcibly override the target for the objcopy tool. +# The cross toolchain is not available in Amazon Linux 2023. +RUN apt-get update && \ + apt-get -y install binutils-aarch64-linux-gnu && \ + ln -sf /usr/bin/aarch64-linux-gnu-objcopy /usr/bin/objcopy && \ + rm -rf /var/lib/apt/lists/* + +FROM --platform=$BUILDPLATFORM jlink-base AS jlink-amd64-on-arm64 +RUN apt-get update && \ + apt-get -y install binutils-x86-64-linux-gnu && \ + ln -sf /usr/bin/x86_64-linux-gnu-objcopy /usr/bin/objcopy && \ + rm -rf /var/lib/apt/lists/* + +FROM --platform=$BUILDPLATFORM jlink-$TARGETARCH-on-$BUILDARCH as jlink +RUN jlink --no-header-files --no-man-pages --compress=2 \ + --module-path=/crossjdk/jmods --strip-debug --add-modules \ + java.base,java.logging,java.xml,jdk.zipfs \ + --output /jlink + +FROM public.ecr.aws/amazonlinux/amazonlinux:2023-minimal AS base +# The launcher script generated by Gradle uses xargs to parse the argument list. +RUN dnf install -y findutils && \ + dnf clean all +# The first layer contains the slimmed down JRE. +COPY --link --from=jlink /jlink /usr/lib/java +ENV JAVA_HOME="/usr/lib/java" PATH="/usr/lib/java/bin:${PATH}" +# Layer with platform-independent dependencies, slow changing. +ADD --link common_lib /app/lib + +FROM base AS base-amd64 +# Layer with platform-dependent dependencies, slow changing. +ADD --link common_amd64_lib /app/lib + +FROM base AS base-arm64 +# Layer with platform-dependent dependencies, slow changing. +ADD --link common_arm64_lib /app/lib + +FROM base-$TARGETARCH + diff --git a/docker/Dockerfile.cli b/docker/Dockerfile.cli new file mode 100644 index 00000000..54396d4e --- /dev/null +++ b/docker/Dockerfile.cli @@ -0,0 +1,26 @@ +# SPDX-FileCopyrightText: 2024 The Refinery Authors +# +# SPDX-License-Identifier: EPL-2.0 + +FROM base AS cli-base +# Layer with platform-dependent dependencies, slow changing. +ADD --link cli_lib /app/lib + +FROM cli-base AS cli-amd64 +# Layer with platform-dependent startup script containing references to all +# dependency versions. +ADD --link cli_amd64_bin /app/bin + +FROM cli-base AS cli-arm64 +# Layer with platform-dependent startup script containing references to all +# dependency versions. +ADD --link cli_arm64_bin /app/bin + +FROM cli-$TARGETARCH +# Layer with platform-independent application jars. +ADD --link cli_app_lib /app/lib +# Common settings added on top. +ENV REFINERY_LIBRARY_PATH=/data +USER 1000 +WORKDIR /data +ENTRYPOINT ["/app/bin/refinery-generator-cli"] diff --git a/docker/Dockerfile.web b/docker/Dockerfile.web new file mode 100644 index 00000000..dd25662e --- /dev/null +++ b/docker/Dockerfile.web @@ -0,0 +1,27 @@ +# SPDX-FileCopyrightText: 2023-2024 The Refinery Authors +# +# SPDX-License-Identifier: EPL-2.0 + +FROM base AS web-base +# Layer with platform-dependent dependencies, slow changing. +ADD --link web_lib /app/lib + +FROM web-base AS web-amd64 +# Layer with platform-dependent startup script containing references to all +# dependency versions. +ADD --link web_amd64_bin /app/bin + +FROM web-base AS web-arm64 +# Layer with platform-dependent startup script containing references to all +# dependency versions. +ADD --link web_arm64_bin /app/bin + +FROM web-$TARGETARCH +# Layer with platform-independent application jars. +ADD --link web_app_lib /app/lib +# Common settings added on top. +ENV REFINERY_LISTEN_HOST=0.0.0.0 REFINERY_LISTEN_PORT=8888 +EXPOSE 8888 +USER 1000 +WORKDIR /app +ENTRYPOINT ["/app/bin/refinery-language-web"] diff --git a/docker/build.sh b/docker/build.sh index 6ddd0cc7..8a03e6d5 100755 --- a/docker/build.sh +++ b/docker/build.sh @@ -1,39 +1,55 @@ #!/usr/bin/env bash -# SPDX-FileCopyrightText: 2023 The Refinery Authors +# SPDX-FileCopyrightText: 2023-2024 The Refinery Authors # # SPDX-License-Identifier: EPL-2.0 set -euo pipefail -(cd .. && ./gradlew :refinery-language-web:distTar) +(cd .. && ./gradlew distTar) refinery_version="$(grep '^version=' ../gradle.properties | cut -d'=' -f2)" -distribution_name="refinery-language-web-${refinery_version}" -rm -rf "${distribution_name}" dist app_lib app_{amd64,arm64}_bin lib lib_{amd64,arm64} +cli_distribution_name="refinery-generator-cli-${refinery_version}" +web_distribution_name="refinery-language-web-${refinery_version}" -tar -xf "../subprojects/language-web/build/distributions/${distribution_name}.tar" -mv "${distribution_name}" dist -mkdir -p app_lib app_{amd64,arm64}_bin lib lib_{amd64,arm64} +rm -rf "${cli_distribution_name}" "${web_distribution_name}" {cli,web}_dist \ + {cli,web}_{,app_}lib common_{,amd64_,arm64_}lib {cli,web}_{amd64,arm64}_bin + +tar -xf "../subprojects/generator-cli/build/distributions/${cli_distribution_name}.tar" +mv "${cli_distribution_name}" cli_dist +tar -xf "../subprojects/language-web/build/distributions/${web_distribution_name}.tar" +mv "${web_distribution_name}" web_dist +mkdir -p {cli,web}_{,app_}lib common_{,amd64_,arm64_}lib {cli,web}_{amd64,arm64}_bin + +# Our application itself is very small, so it will get added as the last layer +# of both containers. +mv cli_dist/lib/refinery-* cli_app_lib +mv web_dist/lib/refinery-* web_app_lib + +for i in cli_dist/lib/*; do + j="web${i#cli}" + if [[ -f "$j" ]]; then + mv "$i" "common_lib${i#cli_dist/lib}" + rm "$j" + fi +done # Move architecture-specific jars to their repsective directories. -mv dist/lib/ortools-linux-x86-64-*.jar lib_amd64 -mv dist/lib/ortools-linux-aarch64-*.jar lib_arm64 -rm dist/lib/ortools-{darwin,win32}-*.jar +mv common_lib/ortools-linux-x86-64-*.jar common_amd64_lib +mv common_lib/ortools-linux-aarch64-*.jar common_arm64_lib +rm common_lib/ortools-{darwin,win32}-*.jar # Move the applications jars for the dependencies into a separate Docker layer # to enable faster updates. -mv dist/lib/refinery-* app_lib -mv dist/lib/* lib +mv cli_dist/lib/* cli_lib +mv web_dist/lib/* web_lib # Omit references to jars not present for the current architecture from the # startup scripts. -sed 's/:\$APP_HOME\/lib\/ortools-\(darwin\|win32\|linux-aarch64\)[^:]\+\.jar//g' dist/bin/refinery-language-web > app_amd64_bin/refinery-language-web -sed 's/:\$APP_HOME\/lib\/ortools-\(darwin\|win32\|linux-x86-64\)[^:]\+\.jar//g' dist/bin/refinery-language-web > app_arm64_bin/refinery-language-web -chmod a+x app_{amd64,arm64}_bin/refinery-language-web -rm -rf dist - -docker buildx build . \ - --platform linux/amd64,linux/arm64 \ - --output "type=image,\"name=ghcr.io/graphs4value/refinery:${refinery_version},ghcr.io/graphs4value/refinery:latest\",push=true,annotation-index.org.opencontainers.image.source=https://github.com/graphs4value/refinery,annotation-index.org.opencontainers.image.description=Refinery: an efficient graph solver for generating well-formed models,annotation-index.org.opencontainers.image.licenses=EPL-2.0" \ - --label 'org.opencontainers.image.source=https://github.com/graphs4value/refinery' \ - --label 'org.opencontainers.image.description=Refinery: an efficient graph solver for generating well-formed models' \ - --label 'org.opencontainers.image.licenses=EPL-2.0' +sed 's/:\$APP_HOME\/lib\/ortools-\(darwin\|win32\|linux-aarch64\)[^:]\+\.jar//g' cli_dist/bin/refinery-generator-cli > cli_amd64_bin/refinery-generator-cli +sed 's/:\$APP_HOME\/lib\/ortools-\(darwin\|win32\|linux-x86-64\)[^:]\+\.jar//g' cli_dist/bin/refinery-generator-cli > cli_arm64_bin/refinery-generator-cli +chmod a+x cli_{amd64,arm64}_bin/refinery-generator-cli +sed 's/:\$APP_HOME\/lib\/ortools-\(darwin\|win32\|linux-aarch64\)[^:]\+\.jar//g' web_dist/bin/refinery-language-web > web_amd64_bin/refinery-language-web +sed 's/:\$APP_HOME\/lib\/ortools-\(darwin\|win32\|linux-x86-64\)[^:]\+\.jar//g' web_dist/bin/refinery-language-web > web_arm64_bin/refinery-language-web +chmod a+x web_{amd64,arm64}_bin/refinery-language-web +rm -rf {cli,web}_dist + +REFINERY_VERSION="${refinery_version}" docker buildx bake -f docker-bake.hcl diff --git a/docker/docker-bake.hcl b/docker/docker-bake.hcl new file mode 100644 index 00000000..b02c82d1 --- /dev/null +++ b/docker/docker-bake.hcl @@ -0,0 +1,39 @@ +# SPDX-FileCopyrightText: 2024 The Refinery Authors +# +# SPDX-License-Identifier: EPL-2.0 + +variable "REFINERY_VERSION" { + default = "" +} + +group "default" { + targets = ["cli", "web"] +} + +target "base" { + dockerfile = "Dockerfile.base" + platforms = ["linux/amd64", "linux/arm64"] + output = ["type=cacheonly"] +} + +target "cli" { + dockerfile = "Dockerfile.cli" + platforms = ["linux/amd64", "linux/arm64"] + output = [ + "type=image,\"name=ghcr.io/graphs4value/refinery:cli-${REFINERY_VERSION},ghcr.io/graphs4value/refinery:cli\",push=true,annotation-index.org.opencontainers.image.source=https://github.com/graphs4value/refinery,annotation-index.org.opencontainers.image.description=Refinery: an efficient graph solver for generating well-formed models,annotation-index.org.opencontainers.image.licenses=EPL-2.0" + ] + contexts = { + base = "target:base" + } +} + +target "web" { + dockerfile = "Dockerfile.web" + platforms = ["linux/amd64", "linux/arm64"] + output = [ + "type=image,\"name=ghcr.io/graphs4value/refinery:${REFINERY_VERSION},ghcr.io/graphs4value/refinery:latest\",push=true,annotation-index.org.opencontainers.image.source=https://github.com/graphs4value/refinery,annotation-index.org.opencontainers.image.description=Refinery: an efficient graph solver for generating well-formed models,annotation-index.org.opencontainers.image.licenses=EPL-2.0" + ] + contexts = { + base = "target:base" + } +} -- cgit v1.2.3-54-g00ecf