From 182d3b3ea433cbed68b4e19ac91acc33d4e331c6 Mon Sep 17 00:00:00 2001
From: Ruffin <roxfoxpox@gmail.com>
Date: Wed, 23 Sep 2020 16:57:08 -0700
Subject: [PATCH] Update Dockerfiles (#1995)
* Revert to simple simlinks
now that image builds are triggered by cron job on CI
rather than repository links on Docker Hub
* Parallelize install/build steps using multi stages
by cascading copies of manifests and workspaces
* Rename stages for installing dependencies
* Remove redundant ARGs and fix WORKDIRs
ARGs do not need to be redeclared for child stages
* Fix typo
* Simplify ARGs and WORKDIRs
* Write package list outside of workspace
to generalize skip up to pattern
* Simplify tests
Assume current WORKDIR
Don't use build mixins for test
* Add parallel stages for testing
use target arg to build added stages, e.g:
`docker build --target=underlay_tester ...`
* Add more comments
* Add simple stage to collect test results
and to easly build all other tester stages in parallel, e.g:
`docker build --build-arg=RUN_TESTS=True --target=workspaces_tester`
---
.dockerhub/debug/Dockerfile | 1 +
.dockerhub/debug/dummy.Dockerfile | 12 ----
.dockerhub/distro.Dockerfile | 9 ++-
.dockerhub/release/Dockerfile | 1 +
.dockerhub/release/dummy.Dockerfile | 12 ----
.dockerhub/source.Dockerfile | 105 +++++++++++++++++++++++-----
.dockerignore | 5 +-
Dockerfile | 7 +-
8 files changed, 99 insertions(+), 53 deletions(-)
create mode 120000 .dockerhub/debug/Dockerfile
delete mode 100644 .dockerhub/debug/dummy.Dockerfile
create mode 120000 .dockerhub/release/Dockerfile
delete mode 100644 .dockerhub/release/dummy.Dockerfile
diff --git a/.dockerhub/debug/Dockerfile b/.dockerhub/debug/Dockerfile
new file mode 120000
index 00000000..36c49d2a
--- /dev/null
+++ b/.dockerhub/debug/Dockerfile
@@ -0,0 +1 @@
+../../Dockerfile
\ No newline at end of file
diff --git a/.dockerhub/debug/dummy.Dockerfile b/.dockerhub/debug/dummy.Dockerfile
deleted file mode 100644
index 121ee27a..00000000
--- a/.dockerhub/debug/dummy.Dockerfile
+++ /dev/null
@@ -1,12 +0,0 @@
-# This is a dummy Dockerfile for setting repository links on Docker Hub.
-# Build rules on Docker Hub can trigger whenever the base image updates.
-# Base images are specified in the FROM: directive in the tracked Dockerfile.
-# However, build args are used by the build hooks to adjust the base image
-# so a single Dockerfile can be reused for multiple CI jobs, reducing maintenance.
-# To re-enable repository linking when using build args in the FROM: directive,
-# this dummy Dockerfile explicitly conveys the base image/repo to link against
-# while build rules that target this still use the same hook in this directory.
-# Note: This only works for non-official images.
-
-FROM osrf/ros2:nightly
-RUN echo "This is a dummy Dockerfile."
\ No newline at end of file
diff --git a/.dockerhub/distro.Dockerfile b/.dockerhub/distro.Dockerfile
index 1f91fd11..47296be4 100644
--- a/.dockerhub/distro.Dockerfile
+++ b/.dockerhub/distro.Dockerfile
@@ -91,11 +91,10 @@ RUN sed --in-place \
# test overlay build
ARG RUN_TESTS
-ARG FAIL_ON_TEST_FAILURE=Ture
+ARG FAIL_ON_TEST_FAILURE=True
RUN if [ -n "$RUN_TESTS" ]; then \
- . $OVERLAY_WS/install/setup.sh && \
- colcon test \
- --mixin $OVERLAY_MIXINS \
- && colcon test-result \
+ . install/setup.sh && \
+ colcon test && \
+ colcon test-result \
|| ([ -z "$FAIL_ON_TEST_FAILURE" ] || exit 1) \
fi
diff --git a/.dockerhub/release/Dockerfile b/.dockerhub/release/Dockerfile
new file mode 120000
index 00000000..36c49d2a
--- /dev/null
+++ b/.dockerhub/release/Dockerfile
@@ -0,0 +1 @@
+../../Dockerfile
\ No newline at end of file
diff --git a/.dockerhub/release/dummy.Dockerfile b/.dockerhub/release/dummy.Dockerfile
deleted file mode 100644
index 81a7ca1c..00000000
--- a/.dockerhub/release/dummy.Dockerfile
+++ /dev/null
@@ -1,12 +0,0 @@
-# This is a dummy Dockerfile for repository links on Docker Hub.
-# Build rules on Docker Hub can trigger whenever the base image updates.
-# Base images are specified in the FROM: directive in the tracked Dockerfile.
-# However, build args are used by the build hooks to adjust the base image
-# so a single Dockerfile can be reused for multiple CI jobs, reducing maintenance.
-# To re-enable repository linking when using build args in the FROM: directive,
-# this dummy Dockerfile explicitly conveys the base image/repo to link against
-# while build rules that target this still use the same hook in this directory.
-# Note: This only works for non-official images.
-
-FROM osrf/ros2:nightly-rmw-nonfree
-RUN echo "This is a dummy Dockerfile."
\ No newline at end of file
diff --git a/.dockerhub/source.Dockerfile b/.dockerhub/source.Dockerfile
index 0d3b6a7b..149bc7c9 100644
--- a/.dockerhub/source.Dockerfile
+++ b/.dockerhub/source.Dockerfile
@@ -12,7 +12,10 @@
# --tag nav2:source \
# --file source.Dockerfile ../
#
-# Omit the `--no-cache` if you know you don't need to break the cache.
+# Use `--no-cache` to break the local docker build cache.
+# Use `--pull` to pull the latest parent image from the remote registry.
+# Use `--target=<stage_name>` to build stages not used for final stage.
+#
# We're only building on top of a ros2 devel image to get the basics
# prerequisites installed such as the apt source, rosdep, etc. We don't want to
# actually use any of the ros release packages. Instead we are going to build
@@ -44,7 +47,7 @@ RUN vcs import ./ < ../underlay.repos && \
ARG OVERLAY_WS
WORKDIR $OVERLAY_WS/src
COPY ./ ./ros-planning/navigation2
-RUN colcon list --names-only | cat > ../packages.txt
+RUN colcon list --names-only | cat >> /opt/packages.txt
# remove skiped packages
WORKDIR /opt
@@ -55,7 +58,7 @@ RUN find ./ \
| xargs dirname | xargs rm -rf || true && \
colcon list --paths-only \
--packages-skip-up-to \
- $(cat $OVERLAY_WS/packages.txt | xargs) \
+ $(cat packages.txt | xargs) \
| xargs rm -rf
# copy manifests for caching
@@ -63,8 +66,8 @@ RUN mkdir -p /tmp/opt && \
find ./ -name "package.xml" | \
xargs cp --parents -t /tmp/opt
-# multi-stage for building
-FROM $FROM_IMAGE AS builder
+# multi-stage for ros2 dependencies
+FROM $FROM_IMAGE AS ros2_depender
ARG DEBIAN_FRONTEND=noninteractive
# edit apt for caching
@@ -97,6 +100,9 @@ RUN --mount=type=cache,target=/var/cache/apt \
$(cat /tmp/skip_keys.txt | xargs) \
"
+# multi-stage for building ros2
+FROM ros2_depender AS ros2_builder
+
# build ros2 source
COPY --from=cacher $ROS2_WS ./
ARG ROS2_MIXINS="release ccache"
@@ -105,13 +111,31 @@ RUN --mount=type=cache,target=/root/.ccache \
--symlink-install \
--mixin $ROS2_MIXINS
+# multi-stage for testing ros2
+FROM ros2_builder AS ros2_tester
+
+# test overlay build
+ARG RUN_TESTS
+ARG FAIL_ON_TEST_FAILURE=True
+RUN if [ -n "$RUN_TESTS" ]; then \
+ . install/setup.sh && \
+ colcon test && \
+ colcon test-result \
+ || ([ -z "$FAIL_ON_TEST_FAILURE" ] || exit 1) \
+ fi
+
+# multi-stage for underlay dependencies
+FROM ros2_depender AS underlay_depender
+
+# copy manifests for caching
+COPY --from=cacher /tmp/$ROS2_WS $ROS2_WS
+
# install underlay dependencies
ARG UNDERLAY_WS
WORKDIR $UNDERLAY_WS
COPY --from=cacher /tmp/$UNDERLAY_WS ./
RUN --mount=type=cache,target=/var/cache/apt \
--mount=type=cache,target=/var/lib/apt \
- . $ROS2_WS/install/setup.sh && \
apt-get update && rosdep install -q -y \
--from-paths src \
$ROS2_WS/src \
@@ -120,6 +144,12 @@ RUN --mount=type=cache,target=/var/cache/apt \
$(cat /tmp/skip_keys.txt | xargs) \
"
+# multi-stage for building underlay
+FROM underlay_depender AS underlay_builder
+
+# copy workspace for caching
+COPY --from=ros2_builder $ROS2_WS $ROS2_WS
+
# build underlay source
COPY --from=cacher $UNDERLAY_WS ./
ARG UNDERLAY_MIXINS="release ccache"
@@ -129,13 +159,32 @@ RUN --mount=type=cache,target=/root/.ccache \
--symlink-install \
--mixin $UNDERLAY_MIXINS
+# multi-stage for testing underlay
+FROM underlay_builder AS underlay_tester
+
+# test overlay build
+ARG RUN_TESTS
+ARG FAIL_ON_TEST_FAILURE=True
+RUN if [ -n "$RUN_TESTS" ]; then \
+ . install/setup.sh && \
+ colcon test && \
+ colcon test-result \
+ || ([ -z "$FAIL_ON_TEST_FAILURE" ] || exit 1) \
+ fi
+
+# multi-stage for overlay dependencies
+FROM underlay_depender AS overlay_depender
+
+# copy manifests for caching
+COPY --from=cacher /tmp/$ROS2_WS $ROS2_WS
+COPY --from=cacher /tmp/$UNDERLAY_WS $UNDERLAY_WS
+
# install overlay dependencies
ARG OVERLAY_WS
WORKDIR $OVERLAY_WS
COPY --from=cacher /tmp/$OVERLAY_WS ./
RUN --mount=type=cache,target=/var/cache/apt \
--mount=type=cache,target=/var/lib/apt \
- . $UNDERLAY_WS/install/setup.sh && \
apt-get update && rosdep install -q -y \
--from-paths src \
$ROS2_WS/src \
@@ -145,6 +194,13 @@ RUN --mount=type=cache,target=/var/cache/apt \
$(cat /tmp/skip_keys.txt | xargs) \
"
+# multi-stage for building overlay
+FROM overlay_depender AS overlay_builder
+
+# copy workspace for caching
+COPY --from=ros2_builder $ROS2_WS $ROS2_WS
+COPY --from=underlay_builder $UNDERLAY_WS $UNDERLAY_WS
+
# build overlay source
COPY --from=cacher $OVERLAY_WS ./
ARG OVERLAY_MIXINS="release ccache"
@@ -154,6 +210,30 @@ RUN --mount=type=cache,target=/root/.ccache \
--symlink-install \
--mixin $OVERLAY_MIXINS
+# multi-stage for testing overlay
+FROM overlay_builder AS overlay_tester
+
+# test overlay build
+ARG RUN_TESTS
+ARG FAIL_ON_TEST_FAILURE=True
+RUN if [ -n "$RUN_TESTS" ]; then \
+ . install/setup.sh && \
+ colcon test && \
+ colcon test-result \
+ || ([ -z "$FAIL_ON_TEST_FAILURE" ] || exit 1) \
+ fi
+
+# multi-stage for testing workspaces
+FROM overlay_builder AS workspaces_tester
+
+# copy workspace test results
+COPY --from=ros2_tester $ROS2_WS/log $ROS2_WS/log
+COPY --from=underlay_tester $UNDERLAY_WS/log $UNDERLAY_WS/log
+COPY --from=overlay_tester $OVERLAY_WS/log $OVERLAY_WS/log
+
+# multi-stage for shipping overlay
+FROM overlay_builder AS overlay_shipper
+
# restore apt for docker
RUN mv /etc/apt/docker-clean /etc/apt/apt.conf.d/ && \
rm -rf /var/lib/apt/lists/
@@ -164,14 +244,3 @@ ENV OVERLAY_WS $OVERLAY_WS
RUN sed --in-place \
's|^source .*|source "$OVERLAY_WS/install/setup.bash"|' \
/ros_entrypoint.sh
-
-# test overlay build
-ARG RUN_TESTS
-ARG FAIL_ON_TEST_FAILURE=Ture
-RUN if [ -n "$RUN_TESTS" ]; then \
- . $OVERLAY_WS/install/setup.sh && \
- colcon test \
- --mixin $OVERLAY_MIXINS \
- && colcon test-result \
- || ([ -z "$FAIL_ON_TEST_FAILURE" ] || exit 1) \
- fi
diff --git a/.dockerignore b/.dockerignore
index ad7d6a52..9e7e1d4a 100644
--- a/.dockerignore
+++ b/.dockerignore
@@ -1,8 +1,9 @@
################################################################################
# Repo
-.circleci/*
-.git/*
+.circleci/
+.dockerhub/
+.git/
.dockerignore
.gitignore
**Dockerfile
diff --git a/Dockerfile b/Dockerfile
index 66067152..ae2b22bb 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -102,9 +102,8 @@ RUN sed --in-place \
ARG RUN_TESTS
ARG FAIL_ON_TEST_FAILURE=True
RUN if [ -n "$RUN_TESTS" ]; then \
- . $OVERLAY_WS/install/setup.sh && \
- colcon test \
- --mixin $OVERLAY_MIXINS \
- && colcon test-result \
+ . install/setup.sh && \
+ colcon test && \
+ colcon test-result \
|| ([ -z "$FAIL_ON_TEST_FAILURE" ] || exit 1) \
fi
--
GitLab