From c52a074aab6d866457f71e217765594ff48009cd Mon Sep 17 00:00:00 2001
From: Joshua Hindmon <joshua.d.hindmon@exxonmobil.com>
Date: Fri, 26 Jan 2024 09:11:38 -0600
Subject: [PATCH] ci/cd work
---
.fossa.yml | 4 +
.gitlab-ci.yml | 21 +++
devops/core-plus/pipeline/override-stages.yml | 123 ++++++++++++++++++
.../cloudbuild/Dockerfile.cloudbuild | 11 +-
4 files changed, 158 insertions(+), 1 deletion(-)
create mode 100644 devops/core-plus/pipeline/override-stages.yml
diff --git a/.fossa.yml b/.fossa.yml
index 53918d386..6ef9aea4e 100644
--- a/.fossa.yml
+++ b/.fossa.yml
@@ -16,6 +16,10 @@ analyze:
type: mvn
target: indexer-core/pom.xml
path: .
+ - name: indexer-core-plus
+ type: mvn
+ target: indexer-core-plus/pom.xml
+ path: .
- name: indexer-aws
type: mvn
target: provider/indexer-aws/pom.xml
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 106e8857b..cabe01eec 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -62,6 +62,7 @@ include:
- local: "devops/gc/pipeline/override-stages.yml"
- local: "devops/aws/pipeline/override-stages.yml"
+ - local: "devops/core-plus/pipeline/override-stages.yml"
.maven:
image: maven:3.9.3-eclipse-temurin-17
@@ -161,6 +162,26 @@ include:
- echo "#!/bin/sh" > $MAVEN_FOREACH
- echo 'xargs -0rI {} $MAVEN_BUILD {} "$@" < $ALL_MAVEN_BUILD_DIRS_Z' >> $MAVEN_FOREACH
+download_plugins:
+ image: maven:3.8.3-openjdk-17-slim
+ stage: build
+ variables:
+ OSM_PACKAGE_REGISTRY_URL: "https://community.opengroup.org/api/v4/projects/1448/packages/maven"
+ OSM_VERSION: "0.25.0-rc2"
+
+ artifacts:
+ paths:
+ - ./tmp/*.jar
+ when: always
+ expire_in: 1 days
+
+ script:
+ - mvn dependency:copy -DrepoUrl=$OSM_PACKAGE_REGISTRY_URL -Dartifact="org.opengroup.osdu:os-osm-postgres:$OSM_VERSION:jar:plugin" -Dtransitive=false -DoutputDirectory="./tmp"
+
+ only:
+ variables:
+ - $PROTECTED == '1'
+
gemnasium-maven-dependency_scanning:
variables:
DS_JAVA_VERSION: 17
diff --git a/devops/core-plus/pipeline/override-stages.yml b/devops/core-plus/pipeline/override-stages.yml
new file mode 100644
index 000000000..ac75040ef
--- /dev/null
+++ b/devops/core-plus/pipeline/override-stages.yml
@@ -0,0 +1,123 @@
+variables:
+ GC_SERVICE: indexer
+ GC_VENDOR: gc
+
+gc-test:
+ variables:
+ CUCUMBER_OPTIONS: "--tags @default"
+ LEGAL_TAG: public-usa-dataset-1
+
+gc-baremetal-test:
+ variables:
+ GC_VENDOR: baremetal
+ CUCUMBER_OPTIONS: "--tags @default"
+ LEGAL_TAG: public-usa-dataset-1
+
+gc-containerize-gitlab:core-plus:
+ stage: containerize
+ needs:
+ - "compile-and-unit-test"
+ - "download_plugins"
+ tags: ["osdu-small"]
+ image: docker:19.03
+ cache: {}
+ variables:
+ BUILD_ARGS: "--build-arg PORT=$GC_PORT"
+ BUILD_PATH: "indexer-core-plus/cloudbuild/Dockerfile"
+ IMAGE_NAME: "indexer-core-plus"
+ script:
+ - !reference [.gc_set_image_name, script]
+ - docker build $EXTRA_TAG -f $BUILD_PATH $BUILD_ARGS .
+ - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
+ - docker push $CI_REGISTRY_IMAGE/$IMAGE_NAME
+ rules:
+ - if: "$CI_COMMIT_BRANCH =~ /^release/"
+ when: never
+ - if: "$CI_COMMIT_TAG"
+ when: never
+ - if: "$PROTECTED == '1'"
+ when: on_success
+
+gc-helm-charts-gitlab:core-plus:
+ stage: containerize
+ tags: ["osdu-small"]
+ needs: ["gc-containerize-gitlab:core-plus"]
+ image:
+ name: alpine/helm:3.11.2
+ entrypoint: [""]
+ variables:
+ IMAGE_NAME: "indexer-core-plus"
+ before_script:
+ - helm registry login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
+ - !reference [.gc_set_image_name, script]
+ - !reference [.gc_substitute_image_in_helm, script]
+ script:
+ - !reference [.gc_define_app_version, script]
+ - helm package $GC_HELM_DEPLOYMENT_DIR -u --version $GC_HELM_PACKAGE_VERSION-$HELM_TAG --app-version $APP_VERSION
+ - helm push $GC_HELM_PACKAGE_NAME-$GC_HELM_PACKAGE_VERSION-$HELM_TAG.tgz oci://$CI_REGISTRY_IMAGE/core-plus-helm
+ rules:
+ - if: "$CI_COMMIT_BRANCH =~ /^release/"
+ when: never
+ - if: "$CI_COMMIT_TAG"
+ when: never
+ - if: "$PROTECTED == '1'"
+ when: on_success
+
+gc-baremetal-deploy:
+ environment:
+ name: GC_Baremetal
+ extends: .gc-baremetal-variables
+ id_tokens:
+ GITLAB_OIDC_TOKEN:
+ aud: https://iam.googleapis.com/projects/${GC_PROJECT_NUMBER}/locations/global/workloadIdentityPools/${GC_POOL_ID}/providers/${GC_PROVIDER_ID}
+ tags: ["osdu-small"]
+ image: gcr.io/google.com/cloudsdktool/cloud-sdk:alpine
+ stage: deploy
+ needs:
+ - "gc-containerize-gitlab:core-plus"
+ - "gc-helm-charts-gitlab:core-plus"
+ retry: 1
+ variables:
+ IMAGE_NAME: "indexer-core-plus"
+ GC_SA_EMAIL: $GC_SA_GKE_EMAIL
+ script:
+ - !reference [.gc_obtain_credentials, script]
+ - !reference [.gc_set_image_name, script]
+ - !reference [.gc_common_config, script]
+ - >
+ helm upgrade $GC_SERVICE-deploy oci://$CI_REGISTRY_IMAGE/core-plus-helm/$GC_HELM_PACKAGE_NAME
+ --version $GC_HELM_PACKAGE_VERSION-$HELM_TAG
+ --install
+ --create-namespace
+ --namespace=$GC_HELM_NAMESPACE
+ --wait
+ --history-max=3
+ --set global.onPremEnabled=true
+ --set global.domain=$GC_DOMAIN
+ --set data.serviceAccountName=$GC_SERVICE
+ --set data.bootstrapServiceAccountName=$GC_BOOTSTRAP_SA
+ --set data.logLevel=INFO
+ --set data.springProfilesActive=$GC_SPRING_PROFILES_ACTIVE
+ --set data.bucketPrefix=$BUCKET_PREFIX
+ --set data.dataPartitionId=$GC_TENANT
+ --set data.groupId=$GROUP_ID
+ --set data.adminUserEmail=$GC_ADMIN_USER_EMAIL
+ --set data.airflowComposerEmail=$GC_AIRFLOW_COMPOSER_EMAIL
+ --set data.sharedTenantName=$GC_TENANT
+ --set opa.data.serviceAccountName=opa
+ --set data.bucketName=$GC_POLICY_BUCKET
+ --set data.googleCloudProject=$GC_PROJECT
+ --set data.subscriberPrivateKeyId=$GC_BAREMETAL_SUBSCRIBER_PRIVATE_KEY_ID
+ $GC_HELM_SETS
+ $GC_HELM_TIMEOUT
+ - !reference [.gc_verify_deploy, script]
+ - !reference [.gc_verify_bootstrap, script]
+
+ rules:
+ - if: "$CI_COMMIT_BRANCH =~ /^release/"
+ when: never
+ - if: "$CI_COMMIT_TAG"
+ when: never
+ - if: "$PROTECTED == '1'"
+ when: on_success
+
diff --git a/indexer-core-plus/cloudbuild/Dockerfile.cloudbuild b/indexer-core-plus/cloudbuild/Dockerfile.cloudbuild
index 101e50f3f..13666dbf7 100644
--- a/indexer-core-plus/cloudbuild/Dockerfile.cloudbuild
+++ b/indexer-core-plus/cloudbuild/Dockerfile.cloudbuild
@@ -8,6 +8,9 @@ ENV PROVIDER_NAME $PROVIDER_NAME
ARG PORT
ENV PORT $PORT
+# Copy plugins
+COPY tmp/os-osm-postgres-*.jar plugins/osm-postgres.jar
+
# Copy the jar to the production image from the builder stage.
COPY provider/indexer-${PROVIDER_NAME}/target/indexer-${PROVIDER_NAME}-*-spring-boot.jar indexer-${PROVIDER_NAME}.jar
@@ -20,4 +23,10 @@ USER 10001:10001
# Run the web service on container startup.
#--add-opens args used to open modules and allow illegal(reflection\private classes and fields) access from the code.
-CMD java --add-opens java.base/java.lang=ALL-UNNAMED --add-opens java.base/java.lang.reflect=ALL-UNNAMED -Djava.security.egd=indexer:/dev/./urandom -Dserver.port=${PORT} -Dlog4j.formatMsgNoLookups=true -jar /app/indexer-${PROVIDER_NAME}.jar
+CMD java --add-opens java.base/java.lang=ALL-UNNAMED \
+ --add-opens java.base/java.lang.reflect=ALL-UNNAMED \
+ -Djava.security.egd=indexer:/dev/./urandom \
+ -Dloader.path=plaugins/ \
+ -Dserver.port=${PORT} \
+ -Dlog4j.formatMsgNoLookups=true \
+ -jar /app/indexer-${PROVIDER_NAME}.jar
--
GitLab