From a22393a67e6d3a53e540f4ff0b378e965be5fa24 Mon Sep 17 00:00:00 2001 From: Bruno Lopes Date: Thu, 11 Apr 2024 10:39:21 -0300 Subject: [PATCH] gitlab-ci: Reduce redundancy on Windows native jobs This commit uses definitions and references to reduce repetition of CI code. Some people (like the GitLab devs?) prefer (almost imposes) this to be done in multiple .yml's using 'worfklow:' but this makes the CI way more complicated and very slow to read. So, let's just keep simple in one human readable file. --- Other changes are: - The custom native arch pipelines now they always generate a .zip package. This helps to ensure that everything is in place and facilitate testing; - The pacman update (-Syyuu) was removed since it is auto done by runners; - The ccache config was moved to .yml. This does NOT affect local builds since meson saves ccache by default for user in '%LocalAppData%/ccache'. --- .gitlab-ci.yml | 258 ++++++------------ build/windows/gitlab-ci/1_build-deps-msys2.sh | 2 + build/windows/gitlab-ci/2_build-gimp-msys2.sh | 18 +- 3 files changed, 90 insertions(+), 188 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index fe9af1045e1..44e595be44f 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -19,9 +19,9 @@ # - GIMP_CI_MSYS2_WIN_AARCH64: trigger the native MSYS2 build for Win Aarch64. # - GIMP_CI_MSYS2_WIN64: trigger the native MSYS2 build for Win 64-bit. # - GIMP_CI_MSYS2_WIN32: trigger the native MSYS2 build for Win 32-bit. +# - GIMP_CI_SOURCES: trigger the Debian build and the source tarball job. # - GIMP_CI_FLATPAK: trigger the (normally) weekly flatpak build and publishing. # - GIMP_CI_WIN_INSTALLER: trigger all native MSYS2 builds then creates Inno Windows installer. -# - GIMP_CI_SOURCES: trigger the Debian gcc build and the source tarball job. # - GIMP_CI_CPPCHECK: trigger cppcheck static analysis. @@ -638,73 +638,72 @@ packaging-win-x86-nightly: ## WINDOWS Aarch64 CI (native MSYS2) ## -deps-win-a64: +.win: rules: # On releases. - if: '$CI_COMMIT_TAG != null' # Custom builds though web GUI, API or schedules. - - if: '$GIMP_CI_MSYS2_WIN_AARCH64 != null' - if: '$GIMP_CI_WIN_INSTALLER != null' # Merge requests with appropriate label. - if: '$CI_MERGE_REQUEST_LABELS =~ /.*5. Windows Installer.*/' - stage: dependencies variables: BUILD_TYPE: "CI_NATIVE" - MSYSTEM: "CLANGARM64" CHERE_INVOKING: "yes" + CCACHE_BASEDIR: "$CI_PROJECT_DIR" + CCACHE_DIR: "$CI_PROJECT_DIR/_ccache" + cache: + key: $CI_JOB_NAME + paths: + - _ccache/ + before_script: + - New-Item -Path .\_ccache -ItemType Directory -Force + artifacts: + name: "${CI_JOB_NAME}-${CI_COMMIT_REF_SLUG}-${CI_COMMIT_SHORT_SHA}" + when: always + +.win-a64: + extends: .win + rules: + # Custom builds though web GUI, API or schedules. + - if: '$GIMP_CI_MSYS2_WIN_AARCH64 != null' tags: - windows-aarch64 + variables: + MSYSTEM: "CLANGARM64" before_script: - # C:\msys64\usr\bin\pacman --noconfirm -Syyuu - script: - # Verification to prevent the job from failing because stuck pacman (ARM only), - # since our runner keeps the status of the MSYS2 installation between jobs + # Verification to prevent the job from failing because stuck pacman (ARM only) - $stuck = Test-Path -Path "C:\msys64\var\lib\pacman\db.lck" -PathType Leaf - if ($stuck -eq "True") { Remove-Item -Force -Path "C:\msys64\var\lib\pacman\db.lck"; Write-Output "Cleaned previous stuck pacman" } - taskkill /F /FI "MODULES eq msys-2.0.dll" - Start-Job -ScriptBlock{sleep 90; taskkill /t /F /IM "pacman.exe"} + +deps-win-a64: + extends: .win-a64 + # GitLab don't support extending 'rules' + rules: + - !reference [.win, rules] + - !reference [.win-a64, rules] + stage: dependencies + script: - C:\msys64\usr\bin\bash -lc "bash -x ./build/windows/gitlab-ci/1_build-deps-msys2.sh" - retry: 2 artifacts: - name: "${CI_JOB_NAME}-${CI_COMMIT_REF_SLUG}-${CI_COMMIT_SHORT_SHA}" - when: always expire_in: 2 hours paths: - _install-a64 needs: [] gimp-win-a64: + extends: .win-a64 + # GitLab don't support extending 'rules' rules: - # On releases. - - if: '$CI_COMMIT_TAG != null' - # Custom builds though web GUI, API or schedules. - - if: '$GIMP_CI_MSYS2_WIN_AARCH64 != null' - - if: '$GIMP_CI_WIN_INSTALLER != null' - # Merge requests with appropriate label. - - if: '$CI_MERGE_REQUEST_LABELS =~ /.*5. Windows Installer.*/' + - !reference [.win, rules] + - !reference [.win-a64, rules] stage: gimp - variables: - BUILD_TYPE: "CI_NATIVE" - MSYSTEM: "CLANGARM64" - CHERE_INVOKING: "yes" - tags: - - windows-aarch64 - before_script: - # C:\msys64\usr\bin\pacman --noconfirm -Syyuu script: - - $stuck = Test-Path -Path "C:\msys64\var\lib\pacman\db.lck" -PathType Leaf - - if ($stuck -eq "True") { - Remove-Item -Force -Path "C:\msys64\var\lib\pacman\db.lck"; - Write-Output "Cleaned previous stuck pacman" } - - taskkill /F /FI "MODULES eq msys-2.0.dll" - - Start-Job -ScriptBlock{sleep 90; taskkill /t /F /IM "pacman.exe"} - C:\msys64\usr\bin\bash -lc "bash -x ./build/windows/gitlab-ci/2_build-gimp-msys2.sh" - retry: 2 artifacts: - name: "${CI_JOB_NAME}-${CI_COMMIT_REF_SLUG}-${CI_COMMIT_SHORT_SHA}" - when: always expire_in: 1 day paths: - _install-a64 @@ -712,42 +711,21 @@ gimp-win-a64: - _build-a64/meson-*/ # Needed by 4_dist-gimp-inno.ps1 in dist-installer-weekly - _build-a64/config.h - cache: - paths: - - _ccache/ needs: ["deps-win-a64"] packaging-win-a64: + extends: .win-a64 + # GitLab don't support extending 'rules' rules: - # On releases. - - if: '$CI_COMMIT_TAG != null' - # Custom builds though web GUI, API or schedules. - - if: '$GIMP_CI_WIN_INSTALLER != null' - # Merge requests with appropriate label. - - if: '$CI_MERGE_REQUEST_LABELS =~ /.*5. Windows Installer.*/' + - !reference [.win, rules] + - !reference [.win-a64, rules] stage: packaging - variables: - BUILD_TYPE: "CI_NATIVE" - MSYSTEM: "CLANGARM64" - CHERE_INVOKING: "yes" - tags: - - windows-aarch64 - before_script: - # C:\msys64\usr\bin\pacman --noconfirm -Syyuu + cache: [] script: - - $stuck = Test-Path -Path "C:\msys64\var\lib\pacman\db.lck" -PathType Leaf - - if ($stuck -eq "True") { - Remove-Item -Force -Path "C:\msys64\var\lib\pacman\db.lck"; - Write-Output "Cleaned previous stuck pacman" } - - taskkill /F /FI "MODULES eq msys-2.0.dll" - - Start-Job -ScriptBlock{sleep 90; taskkill /t /F /IM "pacman.exe"} - C:\msys64\usr\bin\bash -lc "bash -x ./build/windows/gitlab-ci/3_package-gimp-uni_base.sh" - cd gimp-a64 - C:\msys64\usr\bin\bash -lc "bash -x ../build/windows/gitlab-ci/3_package-gimp-uni_sym.sh" - retry: 2 artifacts: - name: "${CI_JOB_NAME}-${CI_COMMIT_REF_SLUG}-${CI_COMMIT_SHORT_SHA}" - when: always expire_in: 1 day paths: - gimp-a64 @@ -760,90 +738,60 @@ packaging-win-a64: ## WINDOWS x86_64 CI (native MSYS2) ## -deps-win-x64: +.win-x64: + extends: .win rules: - # On releases. - - if: '$CI_COMMIT_TAG != null' # Custom builds though web GUI, API or schedules. - if: '$GIMP_CI_MSYS2_WIN64 != null' - - if: '$GIMP_CI_WIN_INSTALLER != null' - # Merge requests with appropriate label. - - if: '$CI_MERGE_REQUEST_LABELS =~ /.*5. Windows Installer.*/' - stage: dependencies - variables: - BUILD_TYPE: "CI_NATIVE" - MSYSTEM: "CLANG64" - CHERE_INVOKING: "yes" tags: - win32-ps - before_script: - - C:\msys64\usr\bin\pacman --noconfirm -Syyuu + variables: + MSYSTEM: "CLANG64" + +deps-win-x64: + extends: .win-x64 + # GitLab don't support extending 'rules' + rules: + - !reference [.win, rules] + - !reference [.win-x64, rules] + stage: dependencies script: - C:\msys64\usr\bin\bash -lc "bash -x ./build/windows/gitlab-ci/1_build-deps-msys2.sh" artifacts: - name: "${CI_JOB_NAME}-${CI_COMMIT_REF_SLUG}-${CI_COMMIT_SHORT_SHA}" - when: always expire_in: 2 hours paths: - _install-x64 needs: [] gimp-win-x64: + extends: .win-x64 + # GitLab don't support extending 'rules' rules: - # On releases. - - if: '$CI_COMMIT_TAG != null' - # Custom builds though web GUI, API or schedules. - - if: '$GIMP_CI_MSYS2_WIN64 != null' - - if: '$GIMP_CI_WIN_INSTALLER != null' - # Merge requests with appropriate label. - - if: '$CI_MERGE_REQUEST_LABELS =~ /.*5. Windows Installer.*/' + - !reference [.win, rules] + - !reference [.win-x64, rules] stage: gimp - variables: - BUILD_TYPE: "CI_NATIVE" - MSYSTEM: "CLANG64" - CHERE_INVOKING: "yes" - tags: - - win32-ps - before_script: - - C:\msys64\usr\bin\pacman --noconfirm -Syyuu script: - C:\msys64\usr\bin\bash -lc "bash -x ./build/windows/gitlab-ci/2_build-gimp-msys2.sh" artifacts: - name: "${CI_JOB_NAME}-${CI_COMMIT_REF_SLUG}-${CI_COMMIT_SHORT_SHA}" - when: always expire_in: 1 day paths: - _install-x64 - - _build-x64/build/windows/installer/ - _build-x64/meson-*/ - cache: - paths: - - _ccache/ needs: ["deps-win-x64"] packaging-win-x64: + extends: .win-x64 + # GitLab don't support extending 'rules' rules: - # On releases. - - if: '$CI_COMMIT_TAG != null' - # Custom builds though web GUI, API or schedules. - - if: '$GIMP_CI_WIN_INSTALLER != null' - # Merge requests with appropriate label. - - if: '$CI_MERGE_REQUEST_LABELS =~ /.*5. Windows Installer.*/' + - !reference [.win, rules] + - !reference [.win-x64, rules] stage: packaging - variables: - MSYSTEM: "CLANG64" - CHERE_INVOKING: "yes" - tags: - - win32-ps - before_script: - - C:\msys64\usr\bin\pacman --noconfirm -Syyuu + cache: [] script: - C:\msys64\usr\bin\bash -lc "bash -x ./build/windows/gitlab-ci/3_package-gimp-uni_base.sh" - cd gimp-x64 - C:\msys64\usr\bin\bash -lc "bash -x ../build/windows/gitlab-ci/3_package-gimp-uni_sym.sh" artifacts: - name: "${CI_JOB_NAME}-${CI_COMMIT_REF_SLUG}-${CI_COMMIT_SHORT_SHA}" - when: always expire_in: 1 day paths: - gimp-x64 @@ -852,89 +800,60 @@ packaging-win-x64: ## WINDOWS x86 CI (native MSYS2) ## -deps-win-x86: +.win-x86: + extends: .win rules: - # On releases. - - if: '$CI_COMMIT_TAG != null' # Custom builds though web GUI, API or schedules. - if: '$GIMP_CI_MSYS2_WIN32 != null' - - if: '$GIMP_CI_WIN_INSTALLER != null' - # Merge requests with appropriate label. - - if: '$CI_MERGE_REQUEST_LABELS =~ /.*5. Windows Installer.*/' - stage: dependencies - variables: - BUILD_TYPE: "CI_NATIVE" - MSYSTEM: "MINGW32" - CHERE_INVOKING: "yes" tags: - win32-ps - before_script: - - C:\msys64\usr\bin\pacman --noconfirm -Syyuu + variables: + MSYSTEM: "MINGW32" + +deps-win-x86: + extends: .win-x86 + # GitLab don't support extending 'rules' + rules: + - !reference [.win, rules] + - !reference [.win-x86, rules] + stage: dependencies script: - C:\msys64\usr\bin\bash -lc "bash -x ./build/windows/gitlab-ci/1_build-deps-msys2.sh" artifacts: - name: "${CI_JOB_NAME}-${CI_COMMIT_REF_SLUG}-${CI_COMMIT_SHORT_SHA}" - when: always expire_in: 2 hours paths: - _install-x86 needs: [] gimp-win-x86: + extends: .win-x86 + # GitLab don't support extending 'rules' rules: - # On releases. - - if: '$CI_COMMIT_TAG != null' - # Custom builds though web GUI, API or schedules. - - if: '$GIMP_CI_MSYS2_WIN32 != null' - - if: '$GIMP_CI_WIN_INSTALLER != null' - # Merge requests with appropriate label. - - if: '$CI_MERGE_REQUEST_LABELS =~ /.*5. Windows Installer.*/' + - !reference [.win, rules] + - !reference [.win-x86, rules] stage: gimp - variables: - BUILD_TYPE: "CI_NATIVE" - MSYSTEM: "MINGW32" - CHERE_INVOKING: "yes" - tags: - - win32-ps - before_script: - - C:\msys64\usr\bin\pacman --noconfirm -Syyuu script: - C:\msys64\usr\bin\bash -lc "bash -x ./build/windows/gitlab-ci/2_build-gimp-msys2.sh" artifacts: - name: "${CI_JOB_NAME}-${CI_COMMIT_REF_SLUG}-${CI_COMMIT_SHORT_SHA}" - when: always expire_in: 1 day paths: - _install-x86 - _build-x86/meson-*/ - cache: - paths: - - _ccache/ needs: ["deps-win-x86"] packaging-win-x86: + extends: .win-x86 + # GitLab don't support extending 'rules' rules: - # On releases. - - if: '$CI_COMMIT_TAG != null' - # Custom builds though web GUI, API or schedules. - - if: '$GIMP_CI_WIN_INSTALLER != null' - # Merge requests with appropriate label. - - if: '$CI_MERGE_REQUEST_LABELS =~ /.*5. Windows Installer.*/' + - !reference [.win, rules] + - !reference [.win-x86, rules] stage: packaging - variables: - MSYSTEM: "MINGW32" - CHERE_INVOKING: "yes" - tags: - - win32-ps - before_script: - - C:\msys64\usr\bin\pacman --noconfirm -Syyuu + cache: [] script: - C:\msys64\usr\bin\bash -lc "bash -x ./build/windows/gitlab-ci/3_package-gimp-uni_base.sh" - cd gimp-x86 - C:\msys64\usr\bin\bash -lc "bash -x ../build/windows/gitlab-ci/3_package-gimp-uni_sym.sh" artifacts: - name: "${CI_JOB_NAME}-${CI_COMMIT_REF_SLUG}-${CI_COMMIT_SHORT_SHA}" - when: always expire_in: 1 day paths: - gimp-x86 @@ -992,18 +911,13 @@ dist-flatpak-weekly: - 'packaging-flatpak-x64' dist-installer-weekly: + extends: .win + # GitLab don't support extending 'rules' rules: - # On releases. - - if: '$CI_COMMIT_TAG != null' - # Custom builds though web GUI, API or schedules. - - if: '$GIMP_CI_WIN_INSTALLER != null' - # Merge requests with appropriate label. - - if: '$CI_MERGE_REQUEST_LABELS =~ /.*5. Windows Installer.*/' - variables: - CHERE_INVOKING: "yes" + - !reference [.win, rules] + stage: distribution tags: - win32-ps - stage: distribution dependencies: - packaging-win-a64 - packaging-win-x64 diff --git a/build/windows/gitlab-ci/1_build-deps-msys2.sh b/build/windows/gitlab-ci/1_build-deps-msys2.sh index 42cfd8ea8ac..28252e11fa6 100644 --- a/build/windows/gitlab-ci/1_build-deps-msys2.sh +++ b/build/windows/gitlab-ci/1_build-deps-msys2.sh @@ -115,3 +115,5 @@ configure_or_build () configure_or_build babl "-Dwith-docs=false" configure_or_build gegl "-Ddocs=false -Dworkshop=true" + +ccache --show-stats \ No newline at end of file diff --git a/build/windows/gitlab-ci/2_build-gimp-msys2.sh b/build/windows/gitlab-ci/2_build-gimp-msys2.sh index ecdcd16b618..4f38bef3987 100644 --- a/build/windows/gitlab-ci/2_build-gimp-msys2.sh +++ b/build/windows/gitlab-ci/2_build-gimp-msys2.sh @@ -61,20 +61,6 @@ export XDG_DATA_DIRS="${GIMP_PREFIX}/share:/usr/share${XDG_DATA_DIRS:+:$XDG_DATA export GI_TYPELIB_PATH="${GIMP_PREFIX}/${LIB_DIR}/${LIB_SUBDIR}girepository-1.0${GI_TYPELIB_PATH:+:$GI_TYPELIB_PATH}" # End of universal variables -if [ ! -d "_ccache" ]; then - mkdir -p _ccache -fi -export CCACHE_BASEDIR="$(pwd)" -export CCACHE_DIR="${CCACHE_BASEDIR}/_ccache" - -# XXX Do not enable ccache this way because it breaks -# gobject-introspection rules. Let's see later for ccache. -# See: https://github.com/msys2/MINGW-packages/issues/9677 -#export CC="ccache gcc" - -#ccache --zero-stats -#ccache --show-stats - git submodule update --init @@ -99,6 +85,8 @@ else ninja install fi +ccache --show-stats + # XXX Functional fix to the problem of non-configured interpreters make_cmd () @@ -148,8 +136,6 @@ if [[ "$BUILD_TYPE" == "CI_NATIVE" ]]; then cd .. - #ccache --show-stats - # XXX Moving back the prefix to be used as artifacts. mv "${GIMP_PREFIX}" . else -- GitLab