Commit 97f8c456 authored by David Diederich's avatar David Diederich
Browse files

Added needs statements to various jobs to create the DAG based build

This should enable later parts of the pipeline to start as soon as their own
dependencies are satisfied, without waiting for the entire stage to complete.
parent e6f0ff81
.maven:
image: maven:3.3.9-jdk-8
tags: ['docker-runner']
variables:
MAVEN_REPO_PATH: "$CI_PROJECT_DIR/.m2/repository"
MAVEN_CLI_OPTS: "--batch-mode --settings=$CI_PROJECT_DIR/.mvn/community-maven.settings.xml"
cache:
paths:
- $MAVEN_REPO_PATH
artifacts:
paths:
- ./target/*.jar
- ./*/target/*.jar
- ./*/*/target/*.jar
- ./*/*/*/target/*.jar
expire_in: 1 day
before_script:
- | # Check for the presence of a maven wrapper script
if [ -e "$CI_PROJECT_DIR/mvnw" ]; then
export MAVEN_BINARY="$CI_PROJECT_DIR/mvnw"
unset MAVEN_CONFIG
mkdir -pv .mvn/wrapper
else
export MAVEN_BINARY="mvn"
fi
- export MAVEN="$MAVEN_BINARY $MAVEN_CLI_OPTS -Dmaven.repo.local=$MAVEN_REPO_PATH"
- echo $MAVEN_BINARY
- echo $MAVEN_CLI_OPTS
- echo $MAVEN_REPO_PATH
- echo $MAVEN
- mkdir -pv $MAVEN_REPO_PATH
- export ALL_MAVEN_BUILD_DIRS_Z=$(tempfile)
# This scans for all pom.xml files that aren't listed as a child pom (as
# determine by the presence of a <parent> tag).
- |
(python | sort -z > $ALL_MAVEN_BUILD_DIRS_Z) <<EOF
from xml.dom.minidom import parse
import os
import sys
allPomFiles = set()
submoduleFiles = set()
for root, dirnames, filenames in os.walk(os.environ['CI_PROJECT_DIR']):
if 'pom.xml' in filenames:
pomFile = os.path.join(root, 'pom.xml')
allPomFiles.add(pomFile)
dom = parse(pomFile)
for modules in dom.getElementsByTagName('modules'):
for module in modules.getElementsByTagName('module'):
submoduleFiles.add(os.path.join(root, module.firstChild.nodeValue, "pom.xml"))
for pomFile in allPomFiles:
if pomFile not in submoduleFiles:
dir = os.path.dirname(pomFile)
sys.stdout.write(dir + '\0')
EOF
- xargs -0rn 1 echo < $ALL_MAVEN_BUILD_DIRS_Z
# --------------------------------------------------------------------------------
# Template
include:
- local: 'build/maven-template.yml'
.maven:
image: maven:3.3.9-jdk-8
tags: ['docker-runner']
variables:
MAVEN_REPO_PATH: "$CI_PROJECT_DIR/.m2/repository"
MAVEN_CLI_OPTS: "--batch-mode --settings=$CI_PROJECT_DIR/.mvn/community-maven.settings.xml"
cache:
paths:
- $MAVEN_REPO_PATH
artifacts:
paths:
- ./target/*.jar
- ./*/target/*.jar
- ./*/*/target/*.jar
- ./*/*/*/target/*.jar
expire_in: 1 day
before_script:
- | # Check for the presence of a maven wrapper script
if [ -e "$CI_PROJECT_DIR/mvnw" ]; then
export MAVEN_BINARY="$CI_PROJECT_DIR/mvnw"
unset MAVEN_CONFIG
mkdir -pv .mvn/wrapper
else
export MAVEN_BINARY="mvn"
fi
- export MAVEN="$MAVEN_BINARY $MAVEN_CLI_OPTS -Dmaven.repo.local=$MAVEN_REPO_PATH"
- echo $MAVEN_BINARY
- echo $MAVEN_CLI_OPTS
- echo $MAVEN_REPO_PATH
- echo $MAVEN
- mkdir -pv $MAVEN_REPO_PATH
- export ALL_MAVEN_BUILD_DIRS_Z=$(tempfile)
- |
(python | sort -z > $ALL_MAVEN_BUILD_DIRS_Z) <<EOF
from xml.dom.minidom import parse
import os
import sys
allPomFiles = set()
submoduleFiles = set()
for root, dirnames, filenames in os.walk(os.environ['CI_PROJECT_DIR']):
if 'pom.xml' in filenames:
pomFile = os.path.join(root, 'pom.xml')
allPomFiles.add(pomFile)
dom = parse(pomFile)
for modules in dom.getElementsByTagName('modules'):
for module in modules.getElementsByTagName('module'):
submoduleFiles.add(os.path.join(root, module.firstChild.nodeValue, "pom.xml"))
for pomFile in allPomFiles:
if pomFile not in submoduleFiles:
dir = os.path.dirname(pomFile)
sys.stdout.write(dir + '\0')
EOF
- xargs -0rn 1 echo < $ALL_MAVEN_BUILD_DIRS_Z
# --------------------------------------------------------------------------------
# Specific Jobs
# This scans for all pom.xml files that aren't listed as a child pom (as
# determine by the presence of a <parent> tag).
# We build and deploy here so that the jar files are available to be downloaded from the
# GitLab package container during subsequent pipeline steps.
#
compile:
compile-and-unit-test:
extends: .maven
stage: build
script:
# First, build and deploy all the independent POM projects that we find
- xargs -0rn 1 -I {} sh -c "cd '{}' && $MAVEN deploy -DskipTests=true || exit 255" < $ALL_MAVEN_BUILD_DIRS_Z
unit-tests:
extends: .maven
stage: test
script:
# Then, run the unit tests on the main project only
- $MAVEN test
include:
- template: Container-Scanning.gitlab-ci.yml
......@@ -23,6 +24,7 @@ include:
aws-containerize:
extends: .aws
stage: containerize
needs: ['compile-and-unit-test']
variables:
LOCAL_IMAGE_TAG: $CI_REGISTRY_IMAGE/$CI_COMMIT_REF_SLUG:$CI_COMMIT_SHA
AWS_IMAGE_TAG_BASE: $AWS_ACCOUNT_ID.dkr.ecr.$AWS_REGION.amazonaws.com/$AWS_ENVIRONMENT-$AWS_APPLICATION_NAME-repository
......@@ -45,15 +47,25 @@ aws-containerize:
aws-update-ecs:
extends: .aws
stage: deploy
needs: ['aws-containerize']
script:
- ECS_CLUSTER_NAME=$(aws cloudformation list-exports --query "Exports[?Name=='$AWS_ENVIRONMENT-$AWS_APPLICATION_NAME-EcsClusterName'].[Value]" --output text --region $AWS_REGION)
- ECS_SERVICE_NAME=$(aws cloudformation list-exports --query "Exports[?Name=='$AWS_ENVIRONMENT-$AWS_APPLICATION_NAME-EcsServiceName'].[Value]" --output text --region $AWS_REGION)
- aws ecs update-service --cluster $ECS_CLUSTER_NAME --service $ECS_SERVICE_NAME --region $AWS_REGION --force-new-deployment
aws-test:
extends: .aws
stage: integration
needs: ['aws-update-ecs']
script:
- echo "Placeholder job, need to add AWS integration test script here"
- /bin/false
# --------------------------------------------------------------------------------
container_scanning:
stage: verify
stage: scan
needs: ['aws-containerize']
environment:
name: AWS
......
include:
- local: 'build/maven-template.yml'
.gcp:
tags: ['docker-runner']
image: google/cloud-sdk
......@@ -18,6 +22,7 @@
gcp-deploy:
extends: .gcp
stage: deploy
needs: ['compile-and-unit-test']
script:
- sed -e "s|ENVIRONMENT|$GCP_ENVIRONMENT|g" "$GCP_BUILD_SUBDIR/src/main/appengine/app.yaml" | tee "$GCP_DEPLOY_DIR/app.yaml"
- cp -v "${GCP_BUILD_SUBDIR}"/target/*-spring-boot.jar "$GCP_DEPLOY_DIR"
......@@ -29,7 +34,8 @@ gcp-test:
- .gcp
- .maven
stage: verify
stage: integration
needs: ['gcp-deploy']
script:
- cd $GCP_INT_TEST_SUBDIR
- |
......
pages:
stage: deploy
tags: ['docker-runner']
image: alpine
stage: publish
needs: ['fossa-report']
tags: ['docker-runner']
artifacts:
paths:
- public
......
include:
- local: 'build/maven-template.yml'
fossa-analyze:
extends: .maven
image: divido2/fossa-cli-utilities:v1.1
stage: test
stage: scan
needs: ['compile-and-unit-test']
tags: ['docker-runner']
only:
variables:
......@@ -23,7 +28,7 @@ fossa-analyze:
# (without consider order).
fossa-check-notice:
image: divido2/fossa-cli:v2.1
stage: verify
stage: attribution
tags: ['docker-runner']
only:
variables:
......@@ -108,7 +113,7 @@ fossa-check-notice:
fossa-report:
image: divido2/fossa-cli:v2.1
stage: verify
stage: attribution
tags: ['docker-runner']
only:
variables:
......
......@@ -6,8 +6,9 @@ include:
# --------------------------------------------------------------------------------
.ultimate-scanner-config:
stage: test
stage: scan
tags: ['docker-runner']
needs: ['compile-and-unit-test']
cache:
paths:
- $CI_PROJECT_DIR/.m2/repository
......
stages:
- build
- test
- containerize
- scan
- deploy
- verify
- integration
- attribution
- publish
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment