Added needs statements to various jobs to create the DAG based build

This should enable later parts of the pipeline to start as soon as their own
dependencies are satisfied, without waiting for the entire stage to complete.

build/maven-template.yml

+.maven:
+  image: maven:3.3.9-jdk-8
+  tags: ['docker-runner']
+  variables:
+    MAVEN_REPO_PATH: "$CI_PROJECT_DIR/.m2/repository"
+    MAVEN_CLI_OPTS: "--batch-mode --settings=$CI_PROJECT_DIR/.mvn/community-maven.settings.xml"
+
+  cache:
+    paths:
+      - $MAVEN_REPO_PATH
+
+  artifacts:
+    paths:
+      - ./target/*.jar
+      - ./*/target/*.jar
+      - ./*/*/target/*.jar
+      - ./*/*/*/target/*.jar
+    expire_in: 1 day
+
+  before_script:
+    - | # Check for the presence of a maven wrapper script
+      if [ -e "$CI_PROJECT_DIR/mvnw" ]; then
+        export MAVEN_BINARY="$CI_PROJECT_DIR/mvnw"
+        unset MAVEN_CONFIG
+        mkdir -pv .mvn/wrapper
+      else
+        export MAVEN_BINARY="mvn"
+      fi
+    - export MAVEN="$MAVEN_BINARY $MAVEN_CLI_OPTS -Dmaven.repo.local=$MAVEN_REPO_PATH"
+    - echo $MAVEN_BINARY
+    - echo $MAVEN_CLI_OPTS
+    - echo $MAVEN_REPO_PATH
+    - echo $MAVEN
+    - mkdir -pv $MAVEN_REPO_PATH
+    - export ALL_MAVEN_BUILD_DIRS_Z=$(tempfile)
+
+    # This scans for all pom.xml files that aren't listed as a child pom (as
+    # determine by the presence of a <parent> tag).
+    - |
+      (python | sort -z > $ALL_MAVEN_BUILD_DIRS_Z) <<EOF
+      from xml.dom.minidom import parse
+      import os
+      import sys
+
+      allPomFiles = set()
+      submoduleFiles = set()
+
+      for root, dirnames, filenames in os.walk(os.environ['CI_PROJECT_DIR']):
+          if 'pom.xml' in filenames:
+              pomFile = os.path.join(root, 'pom.xml')
+              allPomFiles.add(pomFile)
+
+              dom = parse(pomFile)
+              for modules in dom.getElementsByTagName('modules'):
+                  for module in modules.getElementsByTagName('module'):
+                      submoduleFiles.add(os.path.join(root, module.firstChild.nodeValue, "pom.xml"))
+
+      for pomFile in allPomFiles:
+          if pomFile not in submoduleFiles:
+              dir = os.path.dirname(pomFile)
+              sys.stdout.write(dir + '\0')
+      EOF
+    - xargs -0rn 1 echo < $ALL_MAVEN_BUILD_DIRS_Z

build/maven.yml

 
-# --------------------------------------------------------------------------------
-# Template
+include:
+  - local: 'build/maven-template.yml'
 
-.maven:
-  image: maven:3.3.9-jdk-8
-  tags: ['docker-runner']
-  variables:
-    MAVEN_REPO_PATH: "$CI_PROJECT_DIR/.m2/repository"
-    MAVEN_CLI_OPTS: "--batch-mode --settings=$CI_PROJECT_DIR/.mvn/community-maven.settings.xml"
-
-  cache:
-    paths:
-      - $MAVEN_REPO_PATH
-
-  artifacts:
-    paths:
-      - ./target/*.jar
-      - ./*/target/*.jar
-      - ./*/*/target/*.jar
-      - ./*/*/*/target/*.jar
-    expire_in: 1 day
-
-  before_script:
-    - | # Check for the presence of a maven wrapper script
-      if [ -e "$CI_PROJECT_DIR/mvnw" ]; then
-        export MAVEN_BINARY="$CI_PROJECT_DIR/mvnw"
-        unset MAVEN_CONFIG
-        mkdir -pv .mvn/wrapper
-      else
-        export MAVEN_BINARY="mvn"
-      fi
-    - export MAVEN="$MAVEN_BINARY $MAVEN_CLI_OPTS -Dmaven.repo.local=$MAVEN_REPO_PATH"
-    - echo $MAVEN_BINARY
-    - echo $MAVEN_CLI_OPTS
-    - echo $MAVEN_REPO_PATH
-    - echo $MAVEN
-    - mkdir -pv $MAVEN_REPO_PATH
-    - export ALL_MAVEN_BUILD_DIRS_Z=$(tempfile)
-    - |
-      (python | sort -z > $ALL_MAVEN_BUILD_DIRS_Z) <<EOF
-      from xml.dom.minidom import parse
-      import os
-      import sys
-
-      allPomFiles = set()
-      submoduleFiles = set()
-
-      for root, dirnames, filenames in os.walk(os.environ['CI_PROJECT_DIR']):
-          if 'pom.xml' in filenames:
-              pomFile = os.path.join(root, 'pom.xml')
-              allPomFiles.add(pomFile)
-
-              dom = parse(pomFile)
-              for modules in dom.getElementsByTagName('modules'):
-                  for module in modules.getElementsByTagName('module'):
-                      submoduleFiles.add(os.path.join(root, module.firstChild.nodeValue, "pom.xml"))
-
-      for pomFile in allPomFiles:
-          if pomFile not in submoduleFiles:
-              dir = os.path.dirname(pomFile)
-              sys.stdout.write(dir + '\0')
-      EOF
-    - xargs -0rn 1 echo < $ALL_MAVEN_BUILD_DIRS_Z
-
-# --------------------------------------------------------------------------------
-# Specific Jobs
-
-# This scans for all pom.xml files that aren't listed as a child pom (as
-# determine by the presence of a <parent> tag).
-
-# We build and deploy here so that the jar files are available to be downloaded from the
-# GitLab package container during subsequent pipeline steps.
-#
-compile:
+compile-and-unit-test:
   extends: .maven
   stage: build
   script:
+    # First, build and deploy all the independent POM projects that we find
     - xargs -0rn 1 -I {} sh -c "cd '{}' && $MAVEN deploy -DskipTests=true || exit 255" < $ALL_MAVEN_BUILD_DIRS_Z
 
-unit-tests:
-  extends: .maven
-  stage: test
-  script:
+    # Then, run the unit tests on the main project only
     - $MAVEN test

cloud-providers/aws.yml

+
 include:
   - template: Container-Scanning.gitlab-ci.yml
 
@@ -23,6 +24,7 @@ include:
 aws-containerize:
   extends: .aws
   stage: containerize
+  needs: ['compile-and-unit-test']
   variables:
     LOCAL_IMAGE_TAG: $CI_REGISTRY_IMAGE/$CI_COMMIT_REF_SLUG:$CI_COMMIT_SHA
     AWS_IMAGE_TAG_BASE: $AWS_ACCOUNT_ID.dkr.ecr.$AWS_REGION.amazonaws.com/$AWS_ENVIRONMENT-$AWS_APPLICATION_NAME-repository
@@ -45,15 +47,25 @@ aws-containerize:
 aws-update-ecs:
   extends: .aws
   stage: deploy
+  needs: ['aws-containerize']
   script:
     - ECS_CLUSTER_NAME=$(aws cloudformation list-exports --query "Exports[?Name=='$AWS_ENVIRONMENT-$AWS_APPLICATION_NAME-EcsClusterName'].[Value]" --output text --region $AWS_REGION)
     - ECS_SERVICE_NAME=$(aws cloudformation list-exports --query "Exports[?Name=='$AWS_ENVIRONMENT-$AWS_APPLICATION_NAME-EcsServiceName'].[Value]" --output text --region $AWS_REGION)
     - aws ecs update-service --cluster $ECS_CLUSTER_NAME --service $ECS_SERVICE_NAME --region $AWS_REGION --force-new-deployment
 
+aws-test:
+  extends: .aws
+  stage: integration
+  needs: ['aws-update-ecs']
+  script:
+    - echo "Placeholder job, need to add AWS integration test script here"
+    - /bin/false
+
 # --------------------------------------------------------------------------------
 
 container_scanning:
-  stage: verify
+  stage: scan
+  needs: ['aws-containerize']
   environment:
     name: AWS
 

cloud-providers/gcp.yml

+
+include:
+  - local: 'build/maven-template.yml'
+
 .gcp:
   tags: ['docker-runner']
   image: google/cloud-sdk
@@ -18,6 +22,7 @@
 gcp-deploy:
   extends: .gcp
   stage: deploy
+  needs: ['compile-and-unit-test']
   script:
     - sed -e "s|ENVIRONMENT|$GCP_ENVIRONMENT|g" "$GCP_BUILD_SUBDIR/src/main/appengine/app.yaml" | tee "$GCP_DEPLOY_DIR/app.yaml"
     - cp -v "${GCP_BUILD_SUBDIR}"/target/*-spring-boot.jar "$GCP_DEPLOY_DIR"
@@ -29,7 +34,8 @@ gcp-test:
     - .gcp
     - .maven
 
-  stage: verify
+  stage: integration
+  needs: ['gcp-deploy']
   script:
     - cd $GCP_INT_TEST_SUBDIR
     - |

publishing/pages.yml

 pages:
-  stage: deploy
-  tags: ['docker-runner']
   image: alpine
+  stage: publish
+  needs: ['fossa-report']
+  tags: ['docker-runner']
   artifacts:
     paths:
       - public

scanners/fossa.yml

+
+include:
+  - local: 'build/maven-template.yml'
+
 fossa-analyze:
   extends: .maven
   image: divido2/fossa-cli-utilities:v1.1
-  stage: test
+  stage: scan
+  needs: ['compile-and-unit-test']
   tags: ['docker-runner']
   only:
     variables:
@@ -23,7 +28,7 @@ fossa-analyze:
 # (without consider order).
 fossa-check-notice:
   image: divido2/fossa-cli:v2.1
-  stage: verify
+  stage: attribution
   tags: ['docker-runner']
   only:
     variables:
@@ -108,7 +113,7 @@ fossa-check-notice:
 
 fossa-report:
   image: divido2/fossa-cli:v2.1
-  stage: verify
+  stage: attribution
   tags: ['docker-runner']
   only:
     variables:

scanners/gitlab-ultimate.yml

@@ -6,8 +6,9 @@ include:
 # --------------------------------------------------------------------------------
 
 .ultimate-scanner-config:
-  stage: test
+  stage: scan
   tags: ['docker-runner']
+  needs: ['compile-and-unit-test']
   cache:
     paths:
       - $CI_PROJECT_DIR/.m2/repository

standard-setup.yml

 stages:
   - build
-  - test
   - containerize
+  - scan
   - deploy
-  - verify
+  - integration
+  - attribution
+  - publish