From 3a672391a7538b293cc49968e6a7565d7d48abf5 Mon Sep 17 00:00:00 2001
From: Abderrahim Kitouni <akitouni@gnome.org>
Date: Wed, 5 Aug 2020 11:41:35 +0100
Subject: [PATCH] .gitlab-ci.yml: use a child pipeline for deploying flatpak
 runtimes

This allows other jobs to proceed without waiting for the different
stages of publishing flatpaks

This also makes ppc64le only published for nightly runtimes and not
beta and stable, and moves publishing to ostree to the new deploy
stage
---
 .gitlab-ci.yml              | 193 ++++++++++++++----------------------
 .gitlab-ci/flat-manager.yml |  70 +++++++++++++
 2 files changed, 144 insertions(+), 119 deletions(-)
 create mode 100644 .gitlab-ci/flat-manager.yml

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index eeeab8eddb..3076625089 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -14,11 +14,12 @@ variables:
 stages:
 - track
 - build
+- deploy-config
 - image
+- deploy
 - prepare_flatpak
 - flatpak
 - finish_flatpak
-- ostree
 - reports
 
 default:
@@ -46,31 +47,6 @@ default:
     sourcedir: /cache/buildstream/sources
     EOF
 
-  - |
-    case "$FLATPAK_BRANCH" in
-      master)
-        if [ -n "$NIGHTLY_REPO_TOKEN" ]; then
-          export REPO_TOKEN=$NIGHTLY_REPO_TOKEN
-          export FLAT_MANAGER_SERVER=https://nightly.gnome.org/
-          export FLAT_MANAGER_REPO=nightly
-        fi
-      ;;
-      *beta)
-        if [ -n "$FLATHUB_BETA_REPO_TOKEN" ]; then
-          export REPO_TOKEN=$FLATHUB_BETA_REPO_TOKEN
-          export FLAT_MANAGER_SERVER=https://hub.flathub.org/
-          export FLAT_MANAGER_REPO=beta
-        fi
-      ;;
-      *)
-        if [ -n "$FLATHUB_REPO_TOKEN" ]; then
-          export REPO_TOKEN=$FLATHUB_REPO_TOKEN
-          export FLAT_MANAGER_SERVER=https://hub.flathub.org/
-          export FLAT_MANAGER_REPO=stable
-        fi
-      ;;
-    esac
-
 #
 # Templates shared by multiple jobs
 #
@@ -121,43 +97,15 @@ default:
     - image
     expire_in: '2 days'
 
-.flatpak-template:
-  stage: flatpak
-  dependencies: [track, flatpak-prepare]
-  script:
-  - ostree init --repo repo/ --mode archive
-
-  - ${BST} -o arch "${ARCH}" pull flatpak-runtimes.bst
-  - ${BST} -o arch "${ARCH}" checkout --hardlinks flatpak-runtimes.bst checkout-repo/
-  - flatpak build-commit-from --src-repo=checkout-repo/ repo/
-  - rm -rf checkout-repo/
-
-  - |
-    if test $FLAT_MANAGER_REPO == nightly; then
-        ${BST} -o arch "${ARCH}" pull flatpak-platform-extensions.bst
-        ${BST} -o arch "${ARCH}" checkout --hardlinks flatpak-platform-extensions.bst checkout-repo/
-        flatpak build-commit-from --src-repo=checkout-repo/ repo/
-        rm -rf checkout-repo/
-    fi
-
-  - |
-    if test $FLAT_MANAGER_REPO == nightly -a $ARCH == x86_64; then
-        ${BST} pull openh264-extension.bst:flatpak-repo.bst
-        ${BST} checkout --hardlinks openh264-extension.bst:flatpak-repo.bst checkout-repo/
-        flatpak build-commit-from --src-repo=checkout-repo/ repo/
-        rm -rf checkout-repo/
-    fi
-
-  - flatpak build-update-repo --generate-static-deltas repo/
-
-  - flat-manager-client push $(cat build.txt) repo/
-  rules: &flatpak-publish-rules
+.deploy-rules:
+  rules:
   - if: $EMERGENCY || $BST_TRACK_TAGS != "False"
     when: never
   - if: $CI_COMMIT_REF_NAME == "master" || $CI_COMMIT_REF_NAME =~ /^gnome-\d-\d\d$/
 
 .ostree-template:
-  stage: ostree
+  extends: .deploy-rules
+  stage: deploy
   dependencies: [track]
   script:
   - |
@@ -179,8 +127,6 @@ default:
   - BUILD_ID=$(flat-manager-client create $FLAT_MANAGER_SERVER $FLAT_MANAGER_REPO)
   - flat-manager-client push --commit --publish --wait $BUILD_ID repo/ $(ostree --repo=repo/ refs) || result=$?
   - flat-manager-client purge $BUILD_ID
-  rules: *flatpak-publish-rules
-
 
 #
 # "Real" CI jobs
@@ -193,6 +139,7 @@ track:
   script:
   - 'sed -i "s/track-tags: False/track-tags: ${BST_TRACK_TAGS}/" project.conf'
   - ${BST} track --deps all core.bst flatpak-runtimes.bst vm/image.bst boards/pinebook-pro/image.bst boards/rock64/image.bst boards/raspberrypi-4/image.bst vm/repo-devel.bst
+  - echo $CI_JOB_ID > track-job.id
   retry: 2
   # only run on branches targeting master
   rules:
@@ -200,6 +147,7 @@ track:
   artifacts:
     paths:
     - project.refs
+    - track-job.id
 
 
 build-x86_64:
@@ -228,6 +176,72 @@ build-ppc64le:
   - .ppc64le
 
 
+generate-deploy-config:
+  extends:
+  - .x86_64
+  - .deploy-rules
+  dependencies: [track]
+  stage: deploy-config
+  script:
+  - TRACK_JOB_ID=$(cat track-job.id)
+  - |
+    case "$FLATPAK_BRANCH" in
+      master)
+        FLAT_MANAGER_SERVER=https://nightly.gnome.org/
+        FLAT_MANAGER_REPO=nightly
+        SUPPORTED_ARCHES="x86_64 i686 aarch64 arm ppc64le"
+      ;;
+      *beta)
+        FLAT_MANAGER_SERVER=https://hub.flathub.org/
+        FLAT_MANAGER_REPO=beta
+        SUPPORTED_ARCHES="x86_64 i686 aarch64 arm"
+      ;;
+      *)
+        FLAT_MANAGER_SERVER=https://hub.flathub.org/
+        FLAT_MANAGER_REPO=stable
+        SUPPORTED_ARCHES="x86_64 i686 aarch64 arm"
+      ;;
+    esac
+
+  - |
+    cat > flatpak-variables.yml << END
+    variables:
+      BST: '$BST'
+      TRACK_JOB_ID: '$TRACK_JOB_ID'
+      FLAT_MANAGER_SERVER: '$FLAT_MANAGER_SERVER'
+      FLAT_MANAGER_REPO: '$FLAT_MANAGER_REPO'
+      TARGETS: flatpak-runtimes.bst
+      TARGETS_nightly: flatpak-platform-extensions.bst
+      TARGETS_nightly_x86_64: openh264-extension.bst:flatpak-repo.bst
+      ENVIRONMENT_NAME: flatpak/$FLATPAK_BRANCH
+
+    END
+  - cat flatpak-variables.yml .gitlab-ci/flat-manager.yml > flatpak-deploy.yml
+
+  - |
+    for ARCH in $SUPPORTED_ARCHES; do
+    cat >> flatpak-deploy.yml << END
+
+    flatmgr-$ARCH:
+      extends:
+      - .flatmgr-template
+      - .$ARCH
+    END
+    done
+
+  artifacts:
+    paths:
+    - flatpak-deploy.yml
+
+deploy-flatpak:
+  extends: .deploy-rules
+  stage: deploy
+  trigger:
+    strategy: depend
+    include:
+    - artifact: flatpak-deploy.yml
+      job: generate-deploy-config
+
 vm-image-x86_64:
   extends:
   - .vm-image-template
@@ -295,65 +309,6 @@ raspberrypi-4-image:
   - job: build-aarch64
     artifacts: false
 
-
-flatpak-prepare:
-  extends: .x86_64
-  stage: prepare_flatpak
-  dependencies: []
-  script:
-  - flat-manager-client create $FLAT_MANAGER_SERVER $FLAT_MANAGER_REPO > build.txt
-  artifacts:
-    paths:
-    - build.txt
-  rules: *flatpak-publish-rules
-
-flatpak-x86_64:
-  extends:
-  - .flatpak-template
-  - .x86_64
-
-flatpak-i386:
-  extends:
-  - .flatpak-template
-  - .i686
-
-flatpak-aarch64:
-  extends:
-  - .flatpak-template
-  - .aarch64
-
-flatpak-arm:
-  extends:
-  - .flatpak-template
-  - .arm
-
-flatpak-ppc64le:
-  extends:
-  - .flatpak-template
-  - .ppc64le
-
-flatpak-finish:
-  extends: .x86_64
-  stage: finish_flatpak
-  dependencies: [flatpak-prepare]
-  script:
-  - flat-manager-client commit --wait $(cat build.txt)
-  - flat-manager-client publish --wait $(cat build.txt)
-  - flat-manager-client purge $(cat build.txt)
-  rules: *flatpak-publish-rules
-
-flatpak-finish-failed:
-  extends: .x86_64
-  stage: finish_flatpak
-  dependencies: [flatpak-prepare]
-  script:
-  - flat-manager-client purge $(cat build.txt)
-  rules: &flatpak-failed-rules
-  - if: $EMERGENCY || $BST_TRACK_TAGS != "False"
-    when: never
-  - if: $CI_COMMIT_REF_NAME == "master" || $CI_COMMIT_REF_NAME =~ /^gnome-\d-\d\d$/
-    when: on_failure
-
 ostree-x86_64:
   extends:
   - .ostree-template
diff --git a/.gitlab-ci/flat-manager.yml b/.gitlab-ci/flat-manager.yml
new file mode 100644
index 0000000000..eb3dafc15a
--- /dev/null
+++ b/.gitlab-ci/flat-manager.yml
@@ -0,0 +1,70 @@
+
+include: .gitlab-ci/arch.yml
+
+stages:
+- prepare
+- deploy
+- finish
+
+.flatmgr-template:
+  stage: deploy
+  dependencies: [flatmgr-prepare]
+  script:
+  - test -n "$BUILD_TARGETS" && $BST -o arch $ARCH build $BUILD_TARGETS
+
+  - TARGETS_REPO=TARGETS_${FLAT_MANAGER_REPO}
+  - TARGETS_REPO_ARCH=TARGETS_{$FLAT_MANAGER_REPO}_${ARCH}
+  - TARGETS="$TARGETS ${!TARGETS_REPO} ${!TARGETS_REPO_ARCH}"
+
+  - $BST -o arch $ARCH pull $TARGETS
+
+  - ostree init --repo repo/ --mode archive
+
+  - |
+    for target in $TARGETS; do
+      $BST -o arch $ARCH checkout --hardlinks $target checkout-repo/
+      ostree pull-local --repo repo/ checkout-repo/
+      rm -rf checkout-repo/
+    done
+
+  - flatpak build-update-repo --generate-static-deltas repo/
+  - flat-manager-client push $(cat build.txt) repo/
+  environment:
+    name: $ENVIRONMENT_NAME
+    action: prepare
+
+
+flatmgr-prepare:
+  extends: .x86_64
+  stage: prepare
+  script:
+  - curl -O $CI_PROJECT_URL/-/jobs/$TRACK_JOB_ID/artifacts/raw/project.refs
+  - flat-manager-client create $FLAT_MANAGER_SERVER $FLAT_MANAGER_REPO > build.txt
+  artifacts:
+    paths:
+    - build.txt
+    - project.refs
+  environment:
+    name: $ENVIRONMENT_NAME
+    action: prepare
+
+flatmgr-finish:
+  extends: .x86_64
+  stage: finish
+  dependencies: [flatmgr-prepare]
+  script:
+  - flat-manager-client commit --publish --wait $(cat build.txt)
+  - flat-manager-client publish --wait $(cat build.txt)
+  after_script:
+  - flat-manager-client purge $(cat build.txt)
+  environment:
+    name: $ENVIRONMENT_NAME
+
+flatmgr-finish-failed:
+  extends: .x86_64
+  stage: finish
+  dependencies: [flatmgr-prepare]
+  script:
+  - flat-manager-client purge $(cat build.txt)
+  when: on_failure
+
-- 
GitLab