🔨 Improve Docker local tests support (#25583)

.gitignore

@@ -28,29 +28,11 @@ mczip.h
 *.gen
 *.sublime-workspace
 
-#
 # OS
-#
 applet/
 .DS_Store
 
-#
-# Misc
-#
-*~
-*.orig
-*.rej
-*.bak
-*.idea
-*.i
-*.ii
-*.swp
-tags
-
-#
-# C++
-#
-# Compiled Object files
+# Compiled C++ Object files
 *.slo
 *.lo
 *.o
@@ -81,10 +63,7 @@ tags
 *.out
 *.app
 
-#
-# C
-#
-# Object files
+# Compiled C Object files
 *.o
 *.ko
 *.obj
@@ -144,13 +123,13 @@ vc-fileutils.settings
 .vscode/*
 !.vscode/extensions.json
 
-#Simulation
+# Simulation files
 imgui.ini
 eeprom.dat
 spi_flash.bin
 fs.img
 
-#cmake
+# CMake
 CMakeLists.txt
 src/CMakeLists.txt
 CMakeListsPrivate.txt
@@ -171,3 +150,16 @@ __pycache__
 
 # IOLogger logs
 *_log.csv
+
+# Misc.
+*~
+*.orig
+*.rej
+*.bak
+*.idea
+*.i
+*.ii
+*.swp
+tags
+*.logs
+*.bak

Makefile

@@ -1,11 +1,16 @@
+SCRIPTS_DIR := scripts
+CONTAINER_RT_BIN := docker
+CONTAINER_RT_OPTS := --rm -v $(PWD):/code -v platformio-cache:/root/.platformio
+CONTAINER_IMAGE := marlin-dev
+
 help:
 	@echo "Tasks for local development:"
 	@echo "* tests-single-ci:             Run a single test from inside the CI"
 	@echo "* tests-single-local:          Run a single test locally"
-	@echo "* tests-single-local-docker:   Run a single test locally, using docker-compose"
+	@echo "* tests-single-local-docker:   Run a single test locally, using docker"
 	@echo "* tests-all-local:             Run all tests locally"
-	@echo "* tests-all-local-docker:      Run all tests locally, using docker-compose"
-	@echo "* setup-local-docker:          Setup local docker-compose"
+	@echo "* tests-all-local-docker:      Run all tests locally, using docker"
+	@echo "* setup-local-docker:          Build the local docker image"
 	@echo ""
 	@echo "Options for testing:"
 	@echo "  TEST_TARGET          Set when running tests-single-*, to select the"
@@ -34,19 +39,21 @@ tests-single-local:
 
 tests-single-local-docker:
 	@if ! test -n "$(TEST_TARGET)" ; then echo "***ERROR*** Set TEST_TARGET=<your-module> or use make tests-all-local-docker" ; return 1; fi
-	docker-compose run --rm marlin $(MAKE) tests-single-local TEST_TARGET=$(TEST_TARGET) VERBOSE_PLATFORMIO=$(VERBOSE_PLATFORMIO) GIT_RESET_HARD=$(GIT_RESET_HARD) ONLY_TEST="$(ONLY_TEST)"
+	@if ! $(CONTAINER_RT_BIN) images -q $(CONTAINER_IMAGE) > /dev/null ; then $(MAKE) setup-local-docker ; fi
+	$(CONTAINER_RT_BIN) run $(CONTAINER_RT_OPTS) $(CONTAINER_IMAGE) $(MAKE) tests-single-local TEST_TARGET=$(TEST_TARGET) VERBOSE_PLATFORMIO=$(VERBOSE_PLATFORMIO) GIT_RESET_HARD=$(GIT_RESET_HARD) ONLY_TEST="$(ONLY_TEST)"
 .PHONY: tests-single-local-docker
 
 tests-all-local:
 	export PATH="./buildroot/bin/:./buildroot/tests/:${PATH}" \
 	  && export VERBOSE_PLATFORMIO=$(VERBOSE_PLATFORMIO) \
-	  && for TEST_TARGET in $$(./get_test_targets.py) ; do echo "Running tests for $$TEST_TARGET" ; run_tests . $$TEST_TARGET ; done
+	  && for TEST_TARGET in $$($(SCRIPTS_DIR)/get_test_targets.py) ; do echo "Running tests for $$TEST_TARGET" ; run_tests . $$TEST_TARGET ; done
 .PHONY: tests-all-local
 
 tests-all-local-docker:
-	docker-compose run --rm marlin $(MAKE) tests-all-local VERBOSE_PLATFORMIO=$(VERBOSE_PLATFORMIO) GIT_RESET_HARD=$(GIT_RESET_HARD)
+	@if ! $(CONTAINER_RT_BIN) images -q $(CONTAINER_IMAGE) > /dev/null ; then $(MAKE) setup-local-docker ; fi
+	$(CONTAINER_RT_BIN) run $(CONTAINER_RT_OPTS) $(CONTAINER_IMAGE) $(MAKE) tests-all-local VERBOSE_PLATFORMIO=$(VERBOSE_PLATFORMIO) GIT_RESET_HARD=$(GIT_RESET_HARD)
 .PHONY: tests-all-local-docker
 
 setup-local-docker:
-	docker-compose build
+	$(CONTAINER_RT_BIN) build -t $(CONTAINER_IMAGE) -f docker/Dockerfile .
 .PHONY: setup-local-docker

docker-compose.yml

@@ -1,19 +0,0 @@
-version: "3.8"
-services:
-  # The main image: this doesn't run any particular command, but is mainly used
-  # for running tests locally
-  marlin:
-    image: marlin-dev
-    build:
-      dockerfile: Dockerfile
-      context: docker
-    working_dir: /code
-    volumes:
-    - .:/code
-    - platformio-cache:/root/.platformio
-
-volumes:
-  # This volume holds installed libraries for PlatformIO. If this is deleted you
-  # will have to download all the dependencies again, which can be a very slow
-  # process
-  platformio-cache:

docker/Dockerfile

@@ -1,7 +1,24 @@
 FROM python:3.9.0-buster
 
-RUN pip install -U platformio
+# Disable warnings about not having a TTY
+ARG DEBIAN_FRONTEND=noninteractive
+
+# Disable debconf warnings
+ARG DEBCONF_NOWARNINGS="yes"
+
+# Upgrade pip
+RUN pip install --upgrade pip
+
+# Install platformio toolchain / framework and pyyaml
+RUN pip install -U platformio PyYaml
+
+# Upgrade platformio using development version / branch
 RUN pio upgrade --dev
-# To get the test platforms
-RUN pip install PyYaml
+
+# Set working directory
+WORKDIR /code
+
+# Set volumes / mount points that we are using
+VOLUME /code /root/.platformio
+
 #ENV PATH /code/buildroot/bin/:/code/buildroot/tests/:${PATH}

get_test_targets.py

@@ -1,12 +0,0 @@
-#!/usr/bin/env python
-"""
-Extract the builds used in Github CI, so that we can run them locally
-"""
-import yaml
-
-
-with open('.github/workflows/test-builds.yml') as f:
-    github_configuration = yaml.safe_load(f)
-test_platforms = github_configuration\
-    ['jobs']['test_builds']['strategy']['matrix']['test-platform']
-print(' '.join(test_platforms))

scripts/get_test_targets.py

@@ -0,0 +1,15 @@
+#!/usr/bin/env python
+"""
+Extract the builds used in Github CI, so that we can run them locally
+"""
+import yaml
+
+# Set the yaml file to parse
+yaml_file = '.github/workflows/test-builds.yml'
+
+# Parse the yaml file, and load it into a dictionary (github_configuration)
+with open(yaml_file) as f:
+    github_configuration = yaml.safe_load(f)
+
+# Print out the test platforms
+print(' '.join(github_configuration['jobs']['test_builds']['strategy']['matrix']['test-platform']))