diff --git a/.fossa.yml b/.fossa.yml index ad198294f8c5fb49dd2d00f408493f3993af057c..c530ca18f16acf9fb7f442885d83823722930904 100644 --- a/.fossa.yml +++ b/.fossa.yml @@ -32,3 +32,7 @@ analyze: type: mvn target: provider/legal-aws/pom.xml path: . + - name: legal-ibm + type: mvn + target: provider/legal-ibm/pom.xml + path: . diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 9d280ff22abe139564537ffe29f29cd7aca21a72..bec86ffedd94b7897a55186d807a710ef70090e3 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -6,39 +6,53 @@ variables: GCP_BUILD_SUBDIR: provider/legal-gcp GCP_INT_TEST_SUBDIR: testing/legal-test-gcp - GCP_APPLICATION_NAME: os-legal GCP_ENVIRONMENT: testing GCP_PROJECT: opendes-evt GCP_TENANT_NAME: opendes-evt + OSDU_GCP_BUILD_SUBDIR: provider/legal-gcp + OSDU_GCP_INT_TEST_SUBDIR: testing/legal-test-gcp + OSDU_GCP_APPLICATION_NAME: os-legal + + IBM_BUILD_SUBDIR: provider/legal-ibm + IBM_INT_TEST_SUBDIR: testing/legal-test-ibm + AZURE_SERVICE: legal AZURE_BUILD_SUBDIR: provider/legal-azure AZURE_TEST_SUBDIR: testing/legal-test-azure - IBM_BUILD_SUBDIR: provider/legal-ibm - IBM_INT_TEST_SUBDIR: testing/legal-test-ibm include: - project: "osdu/platform/ci-cd-pipelines" file: "standard-setup.yml" + - project: "osdu/platform/ci-cd-pipelines" file: "build/maven.yml" - - project: 'osdu/platform/ci-cd-pipelines' - ref: "master" - file: "cloud-providers/azure.yml" - - project: 'osdu/platform/ci-cd-pipelines' - ref: "master" - file: 'cloud-providers/gcp.yml' - - project: 'osdu/platform/ci-cd-pipelines' - file: 'cloud-providers/aws.yml' - - project: 'osdu/platform/ci-cd-pipelines' - file: 'cloud-providers/ibm.yml' + - project: "osdu/platform/ci-cd-pipelines" file: "scanners/fossa.yml" + - project: "osdu/platform/ci-cd-pipelines" file: "scanners/gitlab-ultimate.yml" + - project: "osdu/platform/ci-cd-pipelines" + file: "cloud-providers/aws.yml" + + - project: "osdu/platform/ci-cd-pipelines" + file: "cloud-providers/ibm.yml" + + - project: "osdu/platform/ci-cd-pipelines" + file: "cloud-providers/azure.yml" + + - project: 'osdu/platform/ci-cd-pipelines' + ref: "master" + file: 'cloud-providers/osdu-gcp.yml' + + - project: 'osdu/platform/ci-cd-pipelines' + ref: "master" + file: 'cloud-providers/gcp.yml' + compile-and-unit-test: variables: GOOGLE_CLOUD_PROJECT: opendes-evt diff --git a/NOTICE b/NOTICE index 77eb96d603d9e6566f81f2586d76a0c541279cd4..1e31e791c78bbe868be6d24e13f3b7cf644fcac6 100644 --- a/NOTICE +++ b/NOTICE @@ -25,6 +25,8 @@ Apache-2.0 ======================================================================== The following software have components provided under the terms of this license: +- AMQP 1.0 JMS Spring Boot AutoConfiguration (from https://repo1.maven.org/maven2/org/amqphub/spring/amqp-10-jms-spring-boot-autoconfigure) +- AMQP 1.0 JMS Spring Boot Starter (from https://repo1.maven.org/maven2/org/amqphub/spring/amqp-10-jms-spring-boot-starter) - ASM Analysis (from ) - ASM Commons (from ) - ASM Core (from ) @@ -235,6 +237,10 @@ The following software have components provided under the terms of this license: - Apache Commons Text (from http://commons.apache.org/proper/commons-text/) - Apache Commons Validator (from http://commons.apache.org/proper/commons-validator/) - Apache Commons Validator (from http://commons.apache.org/proper/commons-validator/) +- Apache Geronimo JMS Spec 2.0 (from http://geronimo.apache.org/maven/${siteId}/${version}) +- Apache Groovy (from http://groovy-lang.org) +- Apache Groovy (from http://groovy-lang.org) +- Apache Groovy (from http://groovy-lang.org) - Apache HttpAsyncClient (from http://hc.apache.org/httpcomponents-asyncclient) - Apache HttpClient (from http://hc.apache.org/httpcomponents-client) - Apache HttpCore (from http://hc.apache.org/httpcomponents-core-ga) @@ -242,6 +248,7 @@ The following software have components provided under the terms of this license: - Apache Log4j API (from ) - Apache Log4j Core (from ) - Apache Log4j JUL Adapter (from ) +- Apache Log4j SLF4J Binding (from ) - Apache Log4j to SLF4J Adapter (from ) - Apache Maven Invoker (from ) - Apache Maven Reporting Implementation (from ) @@ -266,6 +273,7 @@ The following software have components provided under the terms of this license: - Commons Digester (from http://commons.apache.org/digester/) - Commons Digester (from http://commons.apache.org/digester/) - Commons IO (from http://commons.apache.org/io/) +- Commons IO (from http://commons.apache.org/io/) - Commons Lang (from http://commons.apache.org/lang/) - Converter: Jackson (from ) - Doxia :: APT Module (from ) @@ -298,6 +306,10 @@ The following software have components provided under the terms of this license: - Guava: Google Core Libraries for Java (from https://github.com/google/guava.git) - HPPC Collections (from http://labs.carrotsearch.com) - Hibernate Validator Engine (from ) +- IBM COS Java SDK for Amazon S3 (from https://github.com/ibm/ibm-cos-sdk-java) +- IBM COS Java SDK for COS KMS (from https://github.com/ibm/ibm-cos-sdk-java) +- IBM COS SDK For Java (from https://github.com/ibm/ibm-cos-sdk-java) +- IBM COS SDK for Java - Core (from https://github.com/ibm/ibm-cos-sdk-java) - Identity and Access Management (IAM) API v1-rev247-1.23.0 (from ) - J2ObjC Annotations (from https://github.com/google/j2objc/) - J2ObjC Annotations (from https://github.com/google/j2objc/) @@ -329,6 +341,7 @@ The following software have components provided under the terms of this license: - Java Servlet API (from http://servlet-spec.java.net) - Java UUID Generator (from http://wiki.fasterxml.com/JugHome) - Javassist (from http://www.javassist.org/) +- Javassist (from http://www.javassist.org/) - Jetty Server (from ) - Jetty Utilities (from ) - Joda-Time (from http://www.joda.org/joda-time/) @@ -369,8 +382,7 @@ The following software have components provided under the terms of this license: - Microsoft Application Insights Java SDK Spring Boot starter (from https://github.com/Microsoft/ApplicationInsights-Java) - Microsoft Application Insights Java SDK Web Module (from https://github.com/Microsoft/ApplicationInsights-Java) - Microsoft Application Insights Log4j 2 Appender (from https://github.com/Microsoft/ApplicationInsights-Java) -- Microsoft Application Insights Logback Appender (from https://github.com/Microsoft/ApplicationInsights-Java) -- Mockito (from http://www.mockito.org) +- Microsoft Azure Netty HTTP Client Library (from https://github.com/Azure/azure-sdk-for-java) - Mockito (from http://www.mockito.org) - Mockito (from http://mockito.org) - Mockito (from http://mockito.org) @@ -396,8 +408,10 @@ The following software have components provided under the terms of this license: - OAuth 2.0 SDK with OpenID Connect extensions (from https://bitbucket.org/connect2id/oauth-2.0-sdk-with-openid-connect-extensions) - Objenesis (from http://objenesis.org) - OkHttp (from ) +- OkHttp (from ) - OkHttp Logging Interceptor (from ) - OkHttp URLConnection (from ) +- OkHttp URLConnection (from ) - Okio (from ) - OpenCensus (from https://github.com/census-instrumentation/opencensus-java) - OpenCensus (from https://github.com/census-instrumentation/opencensus-java) @@ -409,6 +423,7 @@ The following software have components provided under the terms of this license: - Plexus Velocity Component (from ) - PowerMock (from http://www.powermock.org) - Protocol Buffer extensions to the Google HTTP Client Library for Java. (from ) +- QpidJMS Client (from ) - Reactive Object Pool (from https://github.com/reactor/reactor-pool) - Reactive Streams Netty driver (from https://github.com/reactor/reactor-netty) - Retrofit (from ) @@ -438,9 +453,12 @@ The following software have components provided under the terms of this license: - Spring Context (from https://github.com/spring-projects/spring-framework) - Spring Core (from https://github.com/spring-projects/spring-framework) - Spring Expression Language (SpEL) (from https://github.com/spring-projects/spring-framework) +- Spring JMS (from https://github.com/spring-projects/spring-framework) +- Spring Messaging (from https://github.com/spring-projects/spring-framework) - Spring Plugin - Core (from ) - Spring Plugin - Metadata Extension (from ) - Spring TestContext Framework (from https://github.com/spring-projects/spring-framework) +- Spring Transaction (from https://github.com/spring-projects/spring-framework) - Spring Web (from https://github.com/spring-projects/spring-framework) - Spring Web MVC (from https://github.com/spring-projects/spring-framework) - T-Digest (from https://github.com/tdunning/t-digest) @@ -473,8 +491,11 @@ The following software have components provided under the terms of this license: - io.grpc:grpc-protobuf-lite (from https://github.com/grpc/grpc-java) - io.grpc:grpc-stub (from https://github.com/grpc/grpc-java) - ion-java (from https://github.com/amznlabs/ion-java/) +- ion-java (from https://github.com/amznlabs/ion-java/) - jackson-databind (from http://github.com/FasterXML/jackson) - jackson-databind (from http://github.com/FasterXML/jackson) +- java-cloudant (from https://cloudant.com) +- java-cloudant (from https://cloudant.com) - javax.inject (from http://code.google.com/p/atinject/) - javax.ws.rs-api (from http://jax-rs-spec.java.net) - jersey-container-servlet (from git://java.net/jersey~code/project/jersey-container-servlet) @@ -496,8 +517,6 @@ The following software have components provided under the terms of this license: - org.xmlunit:xmlunit-core (from http://www.xmlunit.org/) - oro (from ) - parent-join (from https://github.com/elastic/elasticsearch) -- powermock-api-mockito (from ) -- powermock-api-mockito-common (from ) - powermock-api-support (from ) - powermock-core (from http://www.powermock.org) - powermock-module-junit4 (from http://www.powermock.org) @@ -505,6 +524,7 @@ The following software have components provided under the terms of this license: - powermock-reflect (from ) - proto-google-cloud-datastore-v1 (from https://github.com/googleapis/api-client-staging) - proton-j (from ) +- proton-j (from ) - rank-eval (from https://github.com/elastic/elasticsearch) - resilience4j (from https://github.com/resilience4j/resilience4j) - resilience4j (from https://github.com/resilience4j/resilience4j) @@ -519,6 +539,7 @@ The following software have components provided under the terms of this license: - spring-security-oauth2-client (from http://spring.io/spring-security) - spring-security-oauth2-core (from http://spring.io/spring-security) - spring-security-oauth2-jose (from http://spring.io/spring-security) +- spring-security-oauth2-resource-server (from http://spring.io/spring-security) - spring-security-test (from http://spring.io/spring-security) - spring-security-web (from http://spring.io/spring-security) - springfox-core (from https://github.com/springfox/springfox) @@ -572,6 +593,7 @@ The following software have components provided under the terms of this license: - ASM library repackaged as OSGi bundle (from ) - Apache Commons Codec (from http://commons.apache.org/proper/commons-codec/) - Apache Commons Codec (from http://commons.apache.org/proper/commons-codec/) +- Apache Groovy (from http://groovy-lang.org) - GAX (Google Api eXtensions) (from https://github.com/googleapis) - GAX (Google Api eXtensions) (from https://github.com/googleapis) - GAX (Google Api eXtensions) (from https://github.com/googleapis) @@ -588,8 +610,6 @@ The following software have components provided under the terms of this license: - Microsoft Application Insights Java SDK Spring Boot starter (from https://github.com/Microsoft/ApplicationInsights-Java) - Microsoft Application Insights Java SDK Web Module (from https://github.com/Microsoft/ApplicationInsights-Java) - Microsoft Application Insights Log4j 2 Appender (from https://github.com/Microsoft/ApplicationInsights-Java) -- Microsoft Application Insights Logback Appender (from https://github.com/Microsoft/ApplicationInsights-Java) -- Mockito (from http://www.mockito.org) - Mockito (from http://www.mockito.org) - NanoHttpd-Core (from ) - Netty/Codec/HTTP (from ) @@ -634,7 +654,6 @@ The following software have components provided under the terms of this license: - Microsoft Application Insights Java SDK Spring Boot starter (from https://github.com/Microsoft/ApplicationInsights-Java) - Microsoft Application Insights Java SDK Web Module (from https://github.com/Microsoft/ApplicationInsights-Java) - Microsoft Application Insights Log4j 2 Appender (from https://github.com/Microsoft/ApplicationInsights-Java) -- Microsoft Application Insights Logback Appender (from https://github.com/Microsoft/ApplicationInsights-Java) ======================================================================== CC0-1.0 @@ -709,7 +728,6 @@ The following software have components provided under the terms of this license: - Microsoft Application Insights Java SDK Spring Boot starter (from https://github.com/Microsoft/ApplicationInsights-Java) - Microsoft Application Insights Java SDK Web Module (from https://github.com/Microsoft/ApplicationInsights-Java) - Microsoft Application Insights Log4j 2 Appender (from https://github.com/Microsoft/ApplicationInsights-Java) -- Microsoft Application Insights Logback Appender (from https://github.com/Microsoft/ApplicationInsights-Java) - SnakeYAML (from http://www.snakeyaml.org) ======================================================================== @@ -815,13 +833,13 @@ The following software have components provided under the terms of this license: - Java Native Access (from https://github.com/java-native-access/jna) - Java Native Access Platform (from https://github.com/java-native-access/jna) - Javassist (from http://www.javassist.org/) +- Javassist (from http://www.javassist.org/) - Logback Classic Module (from ) - Logback Core Module (from ) - Microsoft Application Insights Java SDK Core (from https://github.com/Microsoft/ApplicationInsights-Java) - Microsoft Application Insights Java SDK Spring Boot starter (from https://github.com/Microsoft/ApplicationInsights-Java) - Microsoft Application Insights Java SDK Web Module (from https://github.com/Microsoft/ApplicationInsights-Java) - Microsoft Application Insights Log4j 2 Appender (from https://github.com/Microsoft/ApplicationInsights-Java) -- Microsoft Application Insights Logback Appender (from https://github.com/Microsoft/ApplicationInsights-Java) - java-getopt (from ) ======================================================================== @@ -831,6 +849,7 @@ The following software have components provided under the terms of this license: - Java Native Access (from https://github.com/java-native-access/jna) - Java Native Access Platform (from https://github.com/java-native-access/jna) +- Javassist (from http://www.javassist.org/) - SnakeYAML (from http://www.snakeyaml.org) ======================================================================== @@ -868,7 +887,6 @@ The following software have components provided under the terms of this license: - Microsoft Application Insights Java SDK Spring Boot starter (from https://github.com/Microsoft/ApplicationInsights-Java) - Microsoft Application Insights Java SDK Web Module (from https://github.com/Microsoft/ApplicationInsights-Java) - Microsoft Application Insights Log4j 2 Appender (from https://github.com/Microsoft/ApplicationInsights-Java) -- Microsoft Application Insights Logback Appender (from https://github.com/Microsoft/ApplicationInsights-Java) - Microsoft Azure Java Core Library (from https://github.com/Azure/azure-sdk-for-java) - Microsoft Azure Netty HTTP Client Library (from https://github.com/Azure/azure-sdk-for-java) - Microsoft Azure SDK annotations (from https://github.com/Microsoft/java-api-annotations) @@ -878,10 +896,9 @@ The following software have components provided under the terms of this license: - Microsoft Azure client library for Identity (from https://github.com/Azure/azure-sdk-for-java) - Microsoft Azure client library for KeyVault Secrets (from https://github.com/Azure/azure-sdk-for-java) - Microsoft Azure common module for Storage (from https://github.com/Azure/azure-sdk-for-java) -- Mockito (from http://www.mockito.org) -- Mockito (from http://mockito.org) - Mockito (from http://mockito.org) - Mockito (from http://www.mockito.org) +- Mockito (from http://mockito.org) - Netty/Codec/HTTP (from ) - Netty/Common (from ) - Plexus :: Default Container (from ) @@ -890,7 +907,6 @@ The following software have components provided under the terms of this license: - SLF4J API Module (from http://www.slf4j.org) - adal4j (from https://github.com/AzureAD/azure-activedirectory-library-for-java) - msal4j (from https://github.com/AzureAD/microsoft-authentication-library-for-java) -- powermock-api-mockito (from ) - spring-security-core (from http://spring.io/spring-security) ======================================================================== @@ -900,12 +916,14 @@ The following software have components provided under the terms of this license: - Cobertura code coverage (from http://cobertura.sourceforge.net) - Javassist (from http://www.javassist.org/) +- Javassist (from http://www.javassist.org/) ======================================================================== MPL-2.0 ======================================================================== The following software have components provided under the terms of this license: +- Javassist (from http://www.javassist.org/) - Javassist (from http://www.javassist.org/) ======================================================================== @@ -960,6 +978,8 @@ public-domain The following software have components provided under the terms of this license: - AWS SDK for Java - Models (from https://aws.amazon.com/sdkforjava) +- Apache Groovy (from http://groovy-lang.org) +- Apache Groovy (from http://groovy-lang.org) - Asynchronous Http Client (from ) - Guava: Google Core Libraries for Java (from https://github.com/google/guava.git) - Guava: Google Core Libraries for Java (from https://github.com/google/guava.git) diff --git a/devops/Jenkinsfile b/devops/Jenkinsfile new file mode 100644 index 0000000000000000000000000000000000000000..edd88f6985b7ed8e0e7c3875cb2cd9ca82b4f099 --- /dev/null +++ b/devops/Jenkinsfile @@ -0,0 +1,74 @@ +pipeline { + agent { + kubernetes { + cloud 'openshift' + label 'maven-openjdk18' + yaml """ +spec: + containers: + - name: jnlp + image: quay.io/openshift/origin-jenkins-agent-maven:v4.0.0 + volumeMounts: + - mountPath: "/tmp" + name: "workspace-volume" + readOnly: false + workingDir: "/tmp" + securityContext: + privileged: false + tty: false + resources: + limits: + cpu: 200m + memory: 2Gi + requests: + cpu: 200m + memory: 2Gi + restartPolicy: "Never" +""" + } + } + + environment { + + //Cluster environment variable(CLS_ENV). Like QA, DEV, PERF, PROD etc. + CLS_ENV = "dev" + + //Service variable(CORE_SERVICE). Like indexer, search, delivery, storage, legal etc. + CORE_SERVICE = "legal" + + //GitHub repo URL credential ID for Environment variable files which saved as Secure text in Jenkins Credential. + GIT_ENV_VAR_PATH_URL = credentials('GitRepo-URL-For-Environment-variables') + + //Personal token variable ID which saved as Secure text in Jenkins Credential. Like: GitHub-PRIVATE-TOKEN. + PRIVATE_TOKEN = credentials('GitHub-PRIVATE-TOKEN') + + def runShell = sh (returnStdout: true, script: "curl --header 'PRIVATE-TOKEN: $PRIVATE_TOKEN' ''$GIT_ENV_VAR_PATH_URL'%2F'$CORE_SERVICE'_'$CLS_ENV'_env.json/raw?ref=master' -s -o env.json") + + } + + stages { + stage('Integration_test') { + environment { + def readContent = readJSON file: 'env.json' + + AUTH_USER_ACCESS = "${readContent['AUTH_USER_ACCESS']}" + AUTH_USER_ACCESS_PASSWORD = "${readContent['AUTH_USER_ACCESS_PASSWORD']}" + KEYCLOAK_CLIENT_ID = "${readContent['KEYCLOAK_CLIENT_ID']}" + KEYCLOAK_CLIENT_SECRET = "${readContent['KEYCLOAK_CLIENT_SECRET']}" + KEYCLOAK_REALM = "${readContent['KEYCLOAK_REALM']}" + KEYCLOAK_URL = "${readContent['KEYCLOAK_URL']}" + HOST_URL = "${readContent['HOST_URL']}" + HOST_URL_dev = "${readContent['HOST_URL_dev']}" + MY_TENANT = "${readContent['MY_TENANT']}" + MY_TENANT_PROJECT = "${readContent['MY_TENANT_PROJECT']}" + + } + steps { + script { + sh 'mvn -f testing/legal-test-ibm/pom.xml test' + } + } + } + } + +} \ No newline at end of file diff --git a/devops/azure/README.md b/devops/azure/README.md new file mode 100644 index 0000000000000000000000000000000000000000..e053a04930fb01908010f2bd7793a06d94846a3d --- /dev/null +++ b/devops/azure/README.md @@ -0,0 +1,21 @@ +# Pipeline Support Commands + +```bash +AZURE_SERVICE="legal" +REPO_BRANCH="master" +TAG="latest" +PARTIAL=${REPO_BRANCH/\//-} +BRANCH=${PARTIAL/./-} + +echo "--set image.branch=$BRANCH --set image.tag=$TAG" + +# Remove the Service +kubectl delete deployment osdu-gitlab-$AZURE_SERVICE +kubectl delete service osdu-gitlab-$AZURE_SERVICE + +# Install the Service +helm upgrade -i osdu-gitlab-$AZURE_SERVICE chart --set image.branch=$BRANCH --set image.tag=$TAG +pod=$(kubectl get pod |grep $AZURE_SERVICE | tail -1 | awk '{print $1}') +status=$(kubectl wait --for=condition=Ready pod/$pod --timeout=60s) +if [[ "$status" != *"met"* ]]; then echo "POD didn't start correctly" ; exit 1 ; fi +``` diff --git a/devops/azure/chart/Chart.yaml b/devops/azure/chart/Chart.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f4bfe007924a4ecb30a3094bce16319d0097ba09 --- /dev/null +++ b/devops/azure/chart/Chart.yaml @@ -0,0 +1,20 @@ +# Copyright © Microsoft Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: v2 +name: legal +appVersion: "latest" +description: Helm Chart for installing legal service. +version: 0.1.0 +type: application diff --git a/devops/azure/chart/templates/deployment.yaml b/devops/azure/chart/templates/deployment.yaml new file mode 100644 index 0000000000000000000000000000000000000000..88d1cd931a3a5b51b6724669b61d7e6962933476 --- /dev/null +++ b/devops/azure/chart/templates/deployment.yaml @@ -0,0 +1,122 @@ +# Copyright © Microsoft Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ .Release.Name }} + namespace: osdu +spec: + replicas: {{ .Values.global.replicaCount }} + selector: + matchLabels: + app: {{ .Chart.Name }} + template: + metadata: + labels: + app: {{ .Chart.Name }} + aadpodidbinding: osdu-identity + spec: + volumes: + - name: azure-keyvault + csi: + driver: secrets-store.csi.k8s.io + readOnly: true + volumeAttributes: + secretProviderClass: azure-keyvault + containers: + - name: {{ .Chart.Name }} + image: {{ .Values.image.repository }}/{{ .Chart.Name }}-{{ .Values.image.branch }}:{{ .Values.image.tag | default .Chart.AppVersion }} + imagePullPolicy: Always + ports: + - containerPort: 80 + readinessProbe: + httpGet: + path: /api/legal/v1/swagger-ui.html + port: 80 + volumeMounts: + - name: azure-keyvault + mountPath: "/mnt/azure-keyvault" + readOnly: true + env: + - name: spring_application_name + value: legal + - name: server.servlet.contextPath + value: /api/legal/v1/ + - name: server_port + value: "80" + - name: ACCEPT_HTTP # TEMPORARY UNTIL HTTPS + value: "true" + - name: KEYVAULT_URI + valueFrom: + configMapKeyRef: + name: osdu-svc-properties + key: ENV_KEYVAULT + - name: AZURE_CLIENT_ID + valueFrom: + secretKeyRef: + name: clientid + key: clientid + - name: AZURE_CLIENT_SECRET + valueFrom: + secretKeyRef: + name: clientpassword + key: clientpassword + - name: AZURE_TENANT_ID + valueFrom: + configMapKeyRef: + name: osdu-svc-properties + key: ENV_TENANT_ID + - name: aad_client_id + valueFrom: + secretKeyRef: + name: appid + key: appid + - name: appinsights_key + valueFrom: + secretKeyRef: + name: appinsights + key: appinsights + - name: servicebus_namespace_name + valueFrom: + configMapKeyRef: + name: osdu-svc-properties + key: ENV_SERVICEBUS_NAMESPACE + - name: storage_account + valueFrom: + configMapKeyRef: + name: osdu-svc-properties + key: ENV_STORAGE_ACCOUNT + - name: azure_activedirectory_session_stateless + value: "true" + - name: azure_activedirectory_AppIdUri + value: "api://$(aad_client_id)" + - name: cosmosdb_database + value: osdu-db + - name: LOG_PREFIX + value: legal + - name: azure_storage_container_name + value: legal-service-azure-configuration + - name: azure_storage_enable_https + value: "true" + - name: legal_service_region + value: us + - name: servicebus_topic_name + value: legaltags + - name: entitlements_service_endpoint + value: http://entitlements-azure/entitlements/v1 + - name: entitlements_service_api_key + value: "OBSOLETE" + - name: azure_istioauth_enabled + value: "true" diff --git a/devops/azure/chart/templates/service.yaml b/devops/azure/chart/templates/service.yaml new file mode 100644 index 0000000000000000000000000000000000000000..eb5d124b64e3fdc13f20c0c0480aa1fd13a7a173 --- /dev/null +++ b/devops/azure/chart/templates/service.yaml @@ -0,0 +1,27 @@ +# Copyright © Microsoft Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: v1 +kind: Service +metadata: + name: {{ .Chart.Name }} + namespace: osdu +spec: + type: ClusterIP + ports: + - protocol: TCP + port: 80 + targetPort: 80 + selector: + app: {{ .Chart.Name }} diff --git a/devops/azure/chart/values.yaml b/devops/azure/chart/values.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3567d0f15392658badcba18a9ee24fd7024baf11 --- /dev/null +++ b/devops/azure/chart/values.yaml @@ -0,0 +1,21 @@ +# Copyright © Microsoft Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +global: + replicaCount: 1 + +image: + repository: community.opengroup.org:5555/osdu/platform/security-and-compliance/legal + branch: master + tag: latest diff --git a/devops/azure/release.yaml b/devops/azure/release.yaml new file mode 100644 index 0000000000000000000000000000000000000000..072fc703aa3a5aeafa81a32d8b77f22e8240220b --- /dev/null +++ b/devops/azure/release.yaml @@ -0,0 +1,151 @@ +--- +# Source: legal/templates/service.yaml +# Copyright © Microsoft Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: v1 +kind: Service +metadata: + name: osdu-gitlab-legal + namespace: osdu +spec: + type: ClusterIP + ports: + - protocol: TCP + port: 80 + targetPort: 80 + selector: + app: osdu-gitlab-legal +--- +# Source: legal/templates/deployment.yaml +# Copyright © Microsoft Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: apps/v1 +kind: Deployment +metadata: + name: osdu-gitlab-legal + namespace: osdu +spec: + replicas: 1 + selector: + matchLabels: + app: osdu-gitlab-legal + template: + metadata: + labels: + app: osdu-gitlab-legal + aadpodidbinding: osdu-identity + spec: + volumes: + - name: azure-keyvault + csi: + driver: secrets-store.csi.k8s.io + readOnly: true + volumeAttributes: + secretProviderClass: azure-keyvault + containers: + - name: osdu-gitlab-legal + image: community.opengroup.org:5555/osdu/platform/security-and-compliance/legal/legal-master:latest + imagePullPolicy: Always + ports: + - containerPort: 80 + readinessProbe: + httpGet: + path: /entitlements/v1/swagger-ui.html + port: 80 + volumeMounts: + - name: azure-keyvault + mountPath: "/mnt/azure-keyvault" + readOnly: true + env: + - name: spring_application_name + value: legal + - name: server.servlet.contextPath + value: /api/legal/v1/ + - name: server_port + value: "80" + - name: ACCEPT_HTTP # TEMPORARY UNTIL HTTPS + value: "true" + - name: KEYVAULT_URI + valueFrom: + configMapKeyRef: + name: osdu-svc-properties + key: ENV_KEYVAULT + - name: AZURE_CLIENT_ID + valueFrom: + secretKeyRef: + name: clientid + key: clientid + - name: AZURE_CLIENT_SECRET + valueFrom: + secretKeyRef: + name: clientpassword + key: clientpassword + - name: AZURE_TENANT_ID + valueFrom: + configMapKeyRef: + name: osdu-svc-properties + key: ENV_TENANT_ID + - name: aad_client_id + valueFrom: + secretKeyRef: + name: appid + key: appid + - name: appinsights_key + valueFrom: + secretKeyRef: + name: appinsights + key: appinsights + - name: servicebus_namespace_name + valueFrom: + configMapKeyRef: + name: osdu-svc-properties + key: ENV_SERVICEBUS_NAMESPACE + - name: storage_account + valueFrom: + configMapKeyRef: + name: osdu-svc-properties + key: ENV_STORAGE_ACCOUNT + - name: azure_activedirectory_session_stateless + value: "true" + - name: azure_activedirectory_AppIdUri + value: "api://$(aad_client_id)" + - name: cosmosdb_database + value: osdu-db + - name: LOG_PREFIX + value: legal + - name: azure_storage_container_name + value: legal-service-azure-configuration + - name: azure_storage_enable_https + value: "true" + - name: legal_service_region + value: us + - name: servicebus_topic_name + value: legaltags + - name: entitlements_service_endpoint + value: http://entitlements-azure/entitlements/v1 + - name: entitlements_service_api_key + value: "OBSOLETE" diff --git a/provider/legal-aws/.env.template b/provider/legal-aws/.env.template index 5d2c69fffc6076391f9ee3299a529599ba95f0a2..4d1c43d48937a2e3b0012b88c10c18482eaa0301 100644 --- a/provider/legal-aws/.env.template +++ b/provider/legal-aws/.env.template @@ -35,29 +35,31 @@ ##### Authentication / Secrets ##### # Replace placeholder text with your own AWS secret access keys # and rename to `.env` - do NOT check-in .env with your credentials! Leave it in .gitignore -AWS_ACCESS_KEY_ID= -AWS_SECRET_KEY= +AWS_ACCESS_KEY_ID=<YOUR_ACCESS_KEY_ID> +AWS_SECRET_KEY=<YOUR_SECRET_KEY> ##### URLs/Ports - these values are most likely to change between environments ############# -APPLICATION_PORT= - +APPLICATION_PORT=8080 ##### Other environment variables ########################################################## -JAVA_HEAP_MEMORY= -SNS_TOPIC_NAME= -ENVIRONMENT= -AWS_REGION= -LOG_LEVEL= -SSM_ENABLED= +ENVIRONMENT=dev +AWS_REGION=us-east-1 +LOG_LEVEL=DEBUG +SSM_ENABLED=True ##### Integration test-specific - these are only used for integration tests, not the app ### -AWS_COGNITO_AUTH_FLOW= -AWS_COGNITO_AUTH_PARAMS_PASSWORD= -AWS_COGNITO_AUTH_PARAMS_USER= -AWS_COGNITO_AUTH_PARAMS_USER_NO_ACCESS= -AWS_COGNITO_CLIENT_ID= -HOST_URL= -MY_TENANT= -AWS_S3_ENDPOINT= -AWS_S3_REGION= -SKIP_HTTP_TESTS=true \ No newline at end of file +AWS_COGNITO_CLIENT_ID=<YOUR_COGNITO_CLIENT_ID> +AWS_COGNITO_AUTH_FLOW=USER_PASSWORD_AUTH +AWS_COGNITO_AUTH_PARAMS_USER=<YOUR_AUTHORIZED_USER> +AWS_COGNITO_AUTH_PARAMS_USER_NO_ACCESS=test-user-without-access@testing.com +AWS_COGNITO_AUTH_PARAMS_PASSWORD=<YOUR_AUTHORIZED_USER_PASSWORD> +AWS_S3_ENDPOINT=s3.us-east-1.amazonaws.com +AWS_S3_REGION=us-east-1 +DYNAMO_DB_REGION=us-east-1 +DYNAMO_DB_ENDPOINT=dynamodb.us-east-1.amazonaws.com +HOST_URL=<YOUR_API_URL>/api/legal/v1/ +S3_LEGAL_CONFIG_BUCKET=<YOUR_S3_BUCKET_NAME> +LEGAL_QUEUE=<YOUR_LEGAL_QUEUE_URL> +MY_TENANT=opendes +SKIP_HTTP_TESTS=true +TABLE_PREFIX=<YOUR_TABLE_PREFIX> \ No newline at end of file diff --git a/provider/legal-aws/build-aws/build-info.py b/provider/legal-aws/build-aws/build-info.py new file mode 100644 index 0000000000000000000000000000000000000000..12935424a5a4a5e9368afda46369856e1c56df45 --- /dev/null +++ b/provider/legal-aws/build-aws/build-info.py @@ -0,0 +1,75 @@ +import boto3 +import json +import os +import argparse + +# Create the build-info.json +parser = argparse.ArgumentParser(description="") + +# env - CODEBUILD_SOURCE_VERSION +parser.add_argument("--branch", type=str, help="") + +# env - CODEBUILD_RESOLVED_SOURCE_VERSION +parser.add_argument("--commit", type=str, help="") + +# env - CODEBUILD_BUILD_ID +parser.add_argument("--buildid", type=str, help="") + +# env - CODEBUILD_BUILD_NUMBER +parser.add_argument("--buildnumber", type=str, help="") + +# Get from directory name +parser.add_argument("--reponame", type=str, help="") + +# env OUTPUT_DIR +parser.add_argument("--outdir", type=str, help="") + +# full ecr image and tag, and any other artifacts +parser.add_argument("--artifact", type=str, action="append", help="") + + + +args = parser.parse_args() + +branch = args.branch +commitId = args.commit +buildId = args.buildid +buildNumber = args.buildnumber +repoName = args.reponame +outputDir = args.outdir +artifacts = args.artifact + +buildInfoFilePath = os.path.join(".", outputDir, "build-info.json") + +print(buildInfoFilePath) + +commitArgs = { + "repositoryName": repoName, + "commitId": commitId +} + +commitDetail = { + "commit": "" +} + +# get the commit detail +try: + codecommit = boto3.client("codecommit") + commitDetail = codecommit.get_commit(**commitArgs) +except Exception as e: + print("Getting commit information from codecommit failed") + +buildInfo = { + "branch": branch, + "build-id": buildId, + "build-number": buildNumber, + "repo": repoName, + "artifacts": artifacts, + "commit": commitDetail["commit"] +} +print(json.dumps(buildInfo, sort_keys=True, indent=4)) + +# write the build.json file to dist +f = open(buildInfoFilePath, "w") +f.write(json.dumps(buildInfo, sort_keys=True, indent=4)) +f.close() diff --git a/provider/legal-aws/build-aws/buildspec.yaml b/provider/legal-aws/build-aws/buildspec.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a247ac98de5e32ae7322bc4c89a534a8ac017d32 --- /dev/null +++ b/provider/legal-aws/build-aws/buildspec.yaml @@ -0,0 +1,91 @@ +# Copyright © 2020 Amazon Web Services +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# https://docs.aws.amazon.com/codebuild/latest/userguide/build-spec-ref.html + +# https://docs.aws.amazon.com/codebuild/latest/userguide/build-env-ref-env-vars.html +version: 0.2 + +phases: + install: + runtime-versions: + java: corretto8 + commands: + - if [ $(echo $CODEBUILD_SOURCE_VERSION | grep -c ^refs/heads.*) -eq 1 ]; then echo "Branch name found"; else echo "This build only supports branch builds" && exit 1; fi + - apt-get update -y + - apt-get install -y maven + - java -version + - mvn -version + # - mvn -B clean # .m2 is not created until the first Maven command + - echo $JAVA_HOME #WHY + - mkdir -p /root/.m2 + - cp ./provider/legal-aws/maven/settings.xml /root/.m2/settings.xml # copy the AWS-specific settings.xml to the CodeBuild instance's .m2 folder + - export AWS_ACCOUNT_ID=`aws sts get-caller-identity | grep Account | cut -d':' -f 2 | cut -d'"' -f 2` + - export AWS_OSDU_DEV_MAVEN_AUTH_TOKEN=`aws codeartifact get-authorization-token --domain $AWS_OSDU_DEV_MAVEN_DOMAIN --domain-owner $AWS_ACCOUNT_ID --query authorizationToken --output text` + pre_build: + commands: + - echo "Logging in to Amazon ECR..." + - $(aws ecr get-login --no-include-email --region $AWS_REGION) # authenticate with ECR via the AWS CLI + build: + commands: + - export REPO_NAME=${PWD##*/} + - export OUTPUT_DIR="dist" + - export BRANCH_NAME=`echo ${CODEBUILD_SOURCE_VERSION} | awk '{gsub("refs/heads/","");gsub("\\.","-");gsub("[[:space:]]","-")}1' | sed 's/\//-/g' | awk '{print tolower($0)}'` + - export ECR_TAG=`echo build.${BRANCH_NAME}.${CODEBUILD_BUILD_NUMBER}.${CODEBUILD_RESOLVED_SOURCE_VERSION} | cut -c 1-120` + - export ECR_IMAGE=${ECR_REGISTRY}:${ECR_TAG} + - export ECR_IMAGE_BRANCH_LATEST=${ECR_REGISTRY}:${BRANCH_NAME} + - export INTEGRATION_TEST_OUTPUT=${OUTPUT_DIR}/testing/integration + - export INTEGRATION_TEST_OUTPUT_BIN=${INTEGRATION_TEST_OUTPUT}/bin + - mkdir -p ${OUTPUT_DIR}/bin + - mkdir -p ${OUTPUT_DIR}/testing && mkdir -p ${INTEGRATION_TEST_OUTPUT} && mkdir -p ${INTEGRATION_TEST_OUTPUT}/bin + - echo "Placeholder" >> ${OUTPUT_DIR}/build-info.json # touched so that the output directory has some content incase the build fails so that testing reports are uploaded + - printenv + + - echo "Building primary service assemblies..." + - mvn -B test install -pl legal-core,provider/legal-aws -Ddeployment.environment=prod + +# Suspended until further notice +# - echo "Copying assemblies to dist..." +# - cp ./provider/legal-aws/target/*spring-boot.jar ${OUTPUT_DIR}/bin # copy aws jars +# - cp ./legal-core/target/*.jar ${OUTPUT_DIR}/bin # copy core jar + + - echo "Building integration testing assemblies and gathering artifacts..." + - ./testing/legal-test-aws/build-aws/prepare-dist.sh + + - echo "Building docker image..." + - docker build -f provider/legal-aws/build-aws/Dockerfile -t ${ECR_IMAGE} . + - docker tag ${ECR_IMAGE} ${ECR_IMAGE_BRANCH_LATEST} + - echo "Pushing docker image..." + - docker push ${ECR_IMAGE} + - docker push ${ECR_IMAGE_BRANCH_LATEST} + + - echo "Generate build-info.json" + - | + python provider/legal-aws/build-aws/build-info.py --branch ${CODEBUILD_SOURCE_VERSION} --commit ${CODEBUILD_RESOLVED_SOURCE_VERSION} \ + --buildid ${CODEBUILD_BUILD_ID} --buildnumber ${CODEBUILD_BUILD_NUMBER} --reponame ${REPO_NAME} --outdir ${OUTPUT_DIR} \ + --artifact ${ECR_IMAGE} +reports: + SurefireReports: # CodeBuild will create a report group called "SurefireReports". + files: #Store all of the files + - "legal-core/target/surefire-reports/**/*" + - "provider/legal-aws/target/surefire-reports/**/*" + base-directory: "." # Location of the reports +artifacts: + files: + - "**/*" + base-directory: "dist" + name: ${REPO_NAME}_${BRANCH_NAME}_$(date +%F)_${CODEBUILD_BUILD_NUMBER}.zip +cache: + paths: + - "/root/.m2/**/*" \ No newline at end of file diff --git a/provider/legal-aws/maven/settings.xml b/provider/legal-aws/maven/settings.xml index ee1d404912c936cb02823d845da9005d992bc9be..b8192246c94558d4c2d65ce1caf42871542dd79e 100644 --- a/provider/legal-aws/maven/settings.xml +++ b/provider/legal-aws/maven/settings.xml @@ -1,9 +1,41 @@ <?xml version="1.0" encoding="UTF-8"?> +<!-- + Copyright © 2020 Amazon Web Services + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> + <settings xmlns="http://maven.apache.org/SETTINGS/1.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/SETTINGS/1.0.0 http://maven.apache.org/xsd/settings-1.0.0.xsd"> <profiles> + <profile> + <id>aws-osdu-dev-maven</id> + <activation> + <activeByDefault>true</activeByDefault> + </activation> + <repositories> + <repository> + <id>aws-osdu-dev-maven</id> + <url>${env.AWS_OSDU_DEV_MAVEN_URL}</url> + </repository> + <repository> + <id>gitlab-os-core-common-maven</id> + <url>https://community.opengroup.org/api/v4/projects/67/packages/maven</url> + </repository> + </repositories> + </profile> <profile> <id>credentialsConfiguration</id> <activation> @@ -20,17 +52,25 @@ </profiles> <servers> - <!-- Azure DevOps Artifact Store --> <server> - <id>os-core</id> - <username>${azure.devops.username}</username> - <!-- The generated token was last updated on 01/13/2020 and expires on or before 01/12/2021 --> - <password>${azure.devops.token}</password> + <id>aws-osdu-dev-maven</id> + <username>aws</username> + <password>${env.AWS_OSDU_DEV_MAVEN_AUTH_TOKEN}</password> </server> </servers> + <!-- CodeArtifact doesn't support external repos yet that aren't Maven Central. ETA Q4 2020. --> + <!-- <mirrors> --> + <!-- <mirror> --> + <!-- <id>aws-osdu-dev-maven</id> --> + <!-- <name>aws-osdu-dev-maven</name> --> + <!-- <url>https://osdu-dev-888733619319.d.codeartifact.us-east-1.amazonaws.com/maven/osdu-maven/</url> --> + <!-- <mirrorOf>*,!gitlab-os-core-common-maven</mirrorOf> --> + <!-- </mirror> --> + <!-- </mirrors> --> + <activeProfiles> <activeProfile>credentialsConfiguration</activeProfile> </activeProfiles> -</settings> +</settings> \ No newline at end of file diff --git a/provider/legal-aws/pom.xml b/provider/legal-aws/pom.xml index b7301c50633cc5e4902ccad664e992d8eb0a87cc..96b6c55257788b372b8a9a5688f431010c560b45 100644 --- a/provider/legal-aws/pom.xml +++ b/provider/legal-aws/pom.xml @@ -1,6 +1,6 @@ <?xml version="1.0" encoding="UTF-8"?> <!-- - Copyright © Amazon Web Services + Copyright © 2020 Amazon Web Services Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -39,7 +39,7 @@ <dependency> <groupId>org.opengroup.osdu.core.aws</groupId> <artifactId>os-core-lib-aws</artifactId> - <version>0.2.0</version> + <version>0.3.7</version> </dependency> <dependency> <groupId>org.opengroup.osdu</groupId> diff --git a/provider/legal-aws/src/main/java/org/opengroup/osdu/legal/aws/jobs/LegalTagPublisherImpl.java b/provider/legal-aws/src/main/java/org/opengroup/osdu/legal/aws/jobs/LegalTagPublisherImpl.java index 0d819cc59d608061a61484c73ab5597104d24b84..4f44eba422b066485f26c119b45c11a779ffb30d 100644 --- a/provider/legal-aws/src/main/java/org/opengroup/osdu/legal/aws/jobs/LegalTagPublisherImpl.java +++ b/provider/legal-aws/src/main/java/org/opengroup/osdu/legal/aws/jobs/LegalTagPublisherImpl.java @@ -72,6 +72,9 @@ public class LegalTagPublisherImpl implements ILegalTagPublisher { messageAttributes.put(DpsHeaders.CORRELATION_ID, new MessageAttributeValue() .withDataType("String") .withStringValue(headers.getCorrelationId())); + messageAttributes.put(DpsHeaders.AUTHORIZATION, new MessageAttributeValue() + .withDataType("String") + .withStringValue(headers.getAuthorization())); for (int i = 0; i < tags.getStatusChangedTags().size(); i += BATCH_SIZE){ List<StatusChangedTag> batch = tags.getStatusChangedTags().subList(i, Math.min(tags.getStatusChangedTags().size(), i + BATCH_SIZE)); diff --git a/provider/legal-aws/src/main/resources/application.properties b/provider/legal-aws/src/main/resources/application.properties index 173ca6ba183da0a20cbbaa71591db9cbad1f4167..1eb038d5ed0136c093a80130210a1fe590a9af6c 100644 --- a/provider/legal-aws/src/main/resources/application.properties +++ b/provider/legal-aws/src/main/resources/application.properties @@ -8,7 +8,7 @@ server.port=${APPLICATION_PORT} aws.lambda.get-groups-function-name=${ENVIRONMENT}-os-entitlements-GroupsFunction REGION=${AWS_REGION} -AUTHORIZE_API=notused +AUTHORIZE_API=${ENTITLEMENTS_BASE_URL}/api/entitlements/v1 LEGAL_HOSTNAME=notused CRON_JOB_IP=10.0.0.1 ACCEPT_HTTP=true diff --git a/provider/legal-azure/README.md b/provider/legal-azure/README.md index 9154f0013c4d45c8821c75ef7c3673b6668fdb35..5f4f747f5c3d1371f726679da7c23d575cd75c27 100644 --- a/provider/legal-azure/README.md +++ b/provider/legal-azure/README.md @@ -60,6 +60,7 @@ az keyvault secret show --vault-name $KEY_VAULT_NAME --name $KEY_VAULT_SECRET_NA | `AZURE_TENANT_ID` | `********` | AD tenant to authenticate users from | yes | keyvault secret: `$KEYVAULT_URI/secrets/app-dev-sp-tenant-id` | | `AZURE_CLIENT_SECRET` | `********` | Secret for `$AZURE_CLIENT_ID` | yes | keyvault secret: `$KEYVAULT_URI/secrets/app-dev-sp-password` | | `appinsights_key` | `********` | API Key for App Insights | yes | output of infrastructure deployment | +| `azure_istioauth_enabled` | `true` | Flag to Disable AAD auth | no | -- | **Required to run integration tests** diff --git a/provider/legal-azure/pom.xml b/provider/legal-azure/pom.xml index a03e493dfed25f65522a0484c6f0fae8091d448e..ef840f316eb9eb23781cb64945137967e103b087 100644 --- a/provider/legal-azure/pom.xml +++ b/provider/legal-azure/pom.xml @@ -88,7 +88,7 @@ <dependency> <groupId>org.opengroup.osdu</groupId> <artifactId>core-lib-azure</artifactId> - <version>0.0.11</version> + <version>0.0.29</version> </dependency> <dependency> <groupId>org.projectlombok</groupId> diff --git a/provider/legal-azure/src/main/java/org/opengroup/osdu/legal/azure/di/AzureBootstrapConfig.java b/provider/legal-azure/src/main/java/org/opengroup/osdu/legal/azure/di/AzureBootstrapConfig.java index d5471859361d177826e304c3b0e55d038e018eb0..dd624714c867346b9a9b0b9fcb6889c1e093acd6 100644 --- a/provider/legal-azure/src/main/java/org/opengroup/osdu/legal/azure/di/AzureBootstrapConfig.java +++ b/provider/legal-azure/src/main/java/org/opengroup/osdu/legal/azure/di/AzureBootstrapConfig.java @@ -37,9 +37,6 @@ public class AzureBootstrapConfig { @Value("${azure.servicebus.namespace-name}") private String serviceBusNamespace; - @Value("${azure.cosmosdb.tenant.collection}") - private String tenantCollectionName; - @Value("${azure.cosmosdb.legal.collection}") private String legalCollectionName; @@ -100,9 +97,4 @@ public class AzureBootstrapConfig { public String legalTagsContainer(){ return legalCollectionName; } - - @Bean - public String tenantInfoContainer(){ - return tenantCollectionName; - } } \ No newline at end of file diff --git a/provider/legal-azure/src/main/java/org/opengroup/osdu/legal/azure/di/TenantFactoryImpl.java b/provider/legal-azure/src/main/java/org/opengroup/osdu/legal/azure/di/TenantFactoryImpl.java deleted file mode 100644 index 34a88b0e0ff467c49a0475c69d21e8910b67a2d6..0000000000000000000000000000000000000000 --- a/provider/legal-azure/src/main/java/org/opengroup/osdu/legal/azure/di/TenantFactoryImpl.java +++ /dev/null @@ -1,84 +0,0 @@ -// Copyright © Microsoft Corporation -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package org.opengroup.osdu.legal.azure.di; - -import org.opengroup.osdu.azure.CosmosStore; -import org.opengroup.osdu.core.common.cache.ICache; -import org.opengroup.osdu.core.common.model.http.DpsHeaders; -import org.opengroup.osdu.core.common.provider.interfaces.ITenantFactory; -import org.opengroup.osdu.core.common.model.tenant.TenantInfo; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Component; - -import java.util.Collection; -import java.util.HashMap; -import java.util.Map; - -@Component -public class TenantFactoryImpl implements ITenantFactory { - - @Autowired - private CosmosStore cosmosStore; - - @Autowired - private String tenantInfoContainer; - - @Autowired - private String cosmosDBName; - - @Autowired - private DpsHeaders headers; - - private Map<String, TenantInfo> tenants; - - public boolean exists(String tenantName) - { - if (this.tenants == null) - initTenants(); - return this.tenants.containsKey(tenantName); - } - - public TenantInfo getTenantInfo(String tenantName) { - if (this.tenants == null) - initTenants(); - return this.tenants.get(tenantName); - } - - public Collection<TenantInfo> listTenantInfo() { - if (this.tenants == null) - initTenants(); - return this.tenants.values(); - } - - public <V> ICache<String, V> createCache(String tenantName, String host, int port, int expireTimeSeconds, Class<V> classOfV) - { - return null; - } - - public void flushCache() {} - - private void initTenants() { - this.tenants = new HashMap<>(); - cosmosStore.findAllItems(headers.getPartitionId(), cosmosDBName, tenantInfoContainer, TenantInfoDoc.class).forEach(doc -> { - TenantInfo ti = new TenantInfo(); - String tenantName = doc.getId(); - ti.setName(tenantName); - String complianceRuleSet = doc.getComplianceRuleSet(); - ti.setComplianceRuleSet(complianceRuleSet); - this.tenants.put(tenantName, ti) ; - }); - } - -} diff --git a/provider/legal-azure/src/main/java/org/opengroup/osdu/legal/azure/di/TenantInfoDoc.java b/provider/legal-azure/src/main/java/org/opengroup/osdu/legal/azure/di/TenantInfoDoc.java deleted file mode 100644 index 30145e260f1dc63c1381f43717c43db09740ebb8..0000000000000000000000000000000000000000 --- a/provider/legal-azure/src/main/java/org/opengroup/osdu/legal/azure/di/TenantInfoDoc.java +++ /dev/null @@ -1,27 +0,0 @@ -// Copyright © Microsoft Corporation -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package org.opengroup.osdu.legal.azure.di; - -import lombok.AllArgsConstructor; -import lombok.Data; -import lombok.NoArgsConstructor; - -@Data -@AllArgsConstructor -@NoArgsConstructor -public class TenantInfoDoc { - private String id; - private String complianceRuleSet; -} diff --git a/provider/legal-azure/src/main/java/org/opengroup/osdu/legal/azure/security/AADSecurityConfig.java b/provider/legal-azure/src/main/java/org/opengroup/osdu/legal/azure/security/AADSecurityConfig.java index 1c96f3deff490025c1e2ddc8c8e8575d9ebb2882..cd284dfe01d92b24c5d040c1d90b8297d06975f7 100644 --- a/provider/legal-azure/src/main/java/org/opengroup/osdu/legal/azure/security/AADSecurityConfig.java +++ b/provider/legal-azure/src/main/java/org/opengroup/osdu/legal/azure/security/AADSecurityConfig.java @@ -15,18 +15,22 @@ package org.opengroup.osdu.legal.azure.security; import com.microsoft.azure.spring.autoconfigure.aad.AADAppRoleStatelessAuthenticationFilter; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.security.config.annotation.method.configuration.EnableGlobalMethodSecurity; import org.springframework.security.config.annotation.web.builders.HttpSecurity; import org.springframework.security.config.annotation.web.configuration.EnableWebSecurity; import org.springframework.security.config.annotation.web.configuration.WebSecurityConfigurerAdapter; import org.springframework.security.config.http.SessionCreationPolicy; +import org.springframework.security.core.parameters.P; import org.springframework.security.web.authentication.UsernamePasswordAuthenticationFilter; import javax.inject.Inject; @EnableWebSecurity @EnableGlobalMethodSecurity(prePostEnabled = true) +@ConditionalOnProperty(value = "azure.istio.auth.enabled", havingValue = "false", matchIfMissing = false) public class AADSecurityConfig extends WebSecurityConfigurerAdapter { + @Inject private AADAppRoleStatelessAuthenticationFilter appRoleAuthFilter; diff --git a/provider/legal-azure/src/main/java/org/opengroup/osdu/legal/azure/security/AzureIstioSecurityConfig.java b/provider/legal-azure/src/main/java/org/opengroup/osdu/legal/azure/security/AzureIstioSecurityConfig.java new file mode 100644 index 0000000000000000000000000000000000000000..2de6575c262f49d71fa75ae25a1466e762275b66 --- /dev/null +++ b/provider/legal-azure/src/main/java/org/opengroup/osdu/legal/azure/security/AzureIstioSecurityConfig.java @@ -0,0 +1,35 @@ +// Copyright © Microsoft Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package org.opengroup.osdu.legal.azure.security; + + +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.security.config.annotation.method.configuration.EnableGlobalMethodSecurity; +import org.springframework.security.config.annotation.web.builders.HttpSecurity; +import org.springframework.security.config.annotation.web.configuration.EnableWebSecurity; +import org.springframework.security.config.annotation.web.configuration.WebSecurityConfigurerAdapter; + + +@EnableWebSecurity +@EnableGlobalMethodSecurity(prePostEnabled = true) +@ConditionalOnProperty(value = "azure.istio.auth.enabled", havingValue = "true", matchIfMissing = true) +public class AzureIstioSecurityConfig extends WebSecurityConfigurerAdapter { + + @Override + protected void configure(HttpSecurity http) throws Exception { + http.httpBasic().disable() + .csrf().disable(); //AuthN is disabled. AuthN is handled by sidecar proxy + } +} diff --git a/provider/legal-azure/src/main/resources/application.properties b/provider/legal-azure/src/main/resources/application.properties index 6744890ff4e6ad7db6ab7973f9ae3d0519cfaf4a..dd504c14fc1edd10c31e5612a53afc5f701bbc9f 100644 --- a/provider/legal-azure/src/main/resources/application.properties +++ b/provider/legal-azure/src/main/resources/application.properties @@ -21,14 +21,17 @@ AUTHORIZE_API_KEY=${entitlements_service_api_key} LEGAL_HOSTNAME=notused CRON_JOB_IP=10.0.0.1 -# Azure AD configuration for OpenIDConnect -azure.activedirectory.session-stateless=true -azure.activedirectory.client-id=${aad_client_id} -azure.activedirectory.AppIdUri=api://${azure.activedirectory.client-id} +# Azure AD configuration for OpenIDConnect, commented below settings to disable AAD AuthN , +# Uncomment it In the Istio AUTHN disabled Scenario +#azure.activedirectory.session-stateless=true +#azure.activedirectory.client-id=${aad_client_id} +#azure.activedirectory.AppIdUri=api://${azure.activedirectory.client-id} + +# Istio Auth Enabled +azure.istio.auth.enabled=${azure_istioauth_enabled} # Azure CosmosDB configuration azure.cosmosdb.database=${cosmosdb_database} -azure.cosmosdb.tenant.collection=TenantInfo azure.cosmosdb.legal.collection=LegalTag # Azure Blob Storage configuration @@ -49,4 +52,8 @@ spring.application.name=legal-azure #logging configuration logging.transaction.enabled=true -logging.slf4jlogger.enabled=true \ No newline at end of file +logging.slf4jlogger.enabled=true + +#TenantFactory Configuration +tenantFactoryImpl.required=true +tenantInfo.container.name=TenantInfo diff --git a/provider/legal-gcp/src/main/appengine/app.yaml b/provider/legal-gcp/src/main/appengine/app.yaml index 22df4b203c8223c9df742668103c62e06b91e923..6be9a6e8ed966b9dfda935b84ab69318fc502fa9 100644 --- a/provider/legal-gcp/src/main/appengine/app.yaml +++ b/provider/legal-gcp/src/main/appengine/app.yaml @@ -34,4 +34,5 @@ handlers: env_variables: SPRING_PROFILES_ACTIVE: "ENVIRONMENT" + ENABLE_FULL_BUCKET_NAME: 'ENABLE_FULL_BUCKET_NAME_VAR' \ No newline at end of file diff --git a/provider/legal-gcp/src/main/java/org/opengroup/osdu/legal/countries/StorageReaderFactoryImpl.java b/provider/legal-gcp/src/main/java/org/opengroup/osdu/legal/countries/StorageReaderFactoryImpl.java index 150829919df3a31c3fa753dba0a9ef776a42276c..040d92900ddea325b279f1353f41307c432936b7 100644 --- a/provider/legal-gcp/src/main/java/org/opengroup/osdu/legal/countries/StorageReaderFactoryImpl.java +++ b/provider/legal-gcp/src/main/java/org/opengroup/osdu/legal/countries/StorageReaderFactoryImpl.java @@ -4,13 +4,17 @@ import org.opengroup.osdu.core.common.model.tenant.TenantInfo; import org.opengroup.osdu.legal.provider.interfaces.IStorageReader; import org.opengroup.osdu.legal.provider.interfaces.IStorageReaderFactory; +import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Component; @Component public class StorageReaderFactoryImpl implements IStorageReaderFactory { + @Value("#{new Boolean('${enable.full.bucket.name:false}')}") + private Boolean isFullBucketName; + @Override public IStorageReader getReader(TenantInfo tenant, String projectRegion) { - return new StorageReaderImpl(tenant, projectRegion); + return new StorageReaderImpl(tenant, projectRegion, isFullBucketName); } } diff --git a/provider/legal-gcp/src/main/java/org/opengroup/osdu/legal/countries/StorageReaderImpl.java b/provider/legal-gcp/src/main/java/org/opengroup/osdu/legal/countries/StorageReaderImpl.java index d6b2602f2b8d1d21f165771396a986b45914112c..68a5ec702285ff228ae09b8bb567833a31f53760 100644 --- a/provider/legal-gcp/src/main/java/org/opengroup/osdu/legal/countries/StorageReaderImpl.java +++ b/provider/legal-gcp/src/main/java/org/opengroup/osdu/legal/countries/StorageReaderImpl.java @@ -2,6 +2,7 @@ package org.opengroup.osdu.legal.countries; import com.google.cloud.storage.*; +import java.util.Objects; import org.opengroup.osdu.core.common.model.tenant.TenantInfo; import org.opengroup.osdu.legal.provider.interfaces.IStorageReader; import org.springframework.http.MediaType; @@ -14,16 +15,22 @@ public class StorageReaderImpl implements IStorageReader { private String projectRegion; private Storage storage; - private static final String BUCKET_NAME = "legal-service-configuration"; + protected static final String BUCKET_NAME = "legal-service-configuration"; private static final String FILE_NAME = "Legal_COO.json"; + private Boolean isFullBucketName = false; public StorageReaderImpl(TenantInfo tenantInfo, String projectRegion) { + new StorageReaderImpl(tenantInfo, projectRegion, false); + } + + public StorageReaderImpl(TenantInfo tenantInfo, String projectRegion, Boolean isFullBucketName) { this.tenantInfo = tenantInfo; this.projectRegion = projectRegion; this.storage = getStorage(); - } + this.isFullBucketName = isFullBucketName; + } - @Override + @Override public byte[] readAllBytes() { BlobId blobId = getBlobId(); byte[] content = null; @@ -67,8 +74,11 @@ public class StorageReaderImpl implements IStorageReader { this.storage.create(blobInfo, "".getBytes(UTF_8)); } - private String getTenantBucketName() { + protected String getTenantBucketName() { + if (Objects.nonNull(isFullBucketName) && isFullBucketName) { return this.tenantInfo.getProjectId() + "-" + this.tenantInfo.getName() + "-" + BUCKET_NAME; + } + return this.tenantInfo.getName() + "-" + BUCKET_NAME; } } diff --git a/provider/legal-gcp/src/main/resources/application.properties b/provider/legal-gcp/src/main/resources/application.properties index a9ffebdae689b1d1d115e882e05298a6b967cc9b..bf07a45760f74579a99557bef569fc1ba61d7b67 100644 --- a/provider/legal-gcp/src/main/resources/application.properties +++ b/provider/legal-gcp/src/main/resources/application.properties @@ -8,4 +8,4 @@ server.port=8080 JAVA_HEAP_OPTS=-Xms4096M -Xmx4096M JAVA_GC_OPTS=-XX:+UseG1GC -XX:+UseStringDeduplication -XX:InitiatingHeapOccupancyPercent=45 - +enable.full.bucket.name=false \ No newline at end of file diff --git a/provider/legal-gcp/src/test/java/org/opengroup/osdu/legal/countries/StorageReaderImplTests.java b/provider/legal-gcp/src/test/java/org/opengroup/osdu/legal/countries/StorageReaderImplTests.java index 83d92384787ab5690915adec911e7a0563e9741b..5ddca787acecc9855c7258b1cbc47e806e255571 100644 --- a/provider/legal-gcp/src/test/java/org/opengroup/osdu/legal/countries/StorageReaderImplTests.java +++ b/provider/legal-gcp/src/test/java/org/opengroup/osdu/legal/countries/StorageReaderImplTests.java @@ -13,6 +13,7 @@ import org.opengroup.osdu.core.common.model.tenant.TenantInfo; import static java.nio.charset.StandardCharsets.UTF_8; import static org.junit.Assert.assertEquals; import static org.mockito.Mockito.*; +import static org.opengroup.osdu.legal.countries.StorageReaderImpl.BUCKET_NAME; import static org.powermock.api.mockito.PowerMockito.when; @RunWith(MockitoJUnitRunner.class) @@ -74,4 +75,37 @@ public class StorageReaderImplTests { byte[] bytes = sut.readAllBytes(); assertEquals(expectedBytes, bytes); } + + @Test + public void should_returnFullBucketName_when_IsFullBucketName_is_true() { + when(tenantInfo.getName()).thenReturn("tenant1"); + when(tenantInfo.getProjectId()).thenReturn("projectId1"); + String bucketName = tenantInfo.getProjectId() + "-" + tenantInfo.getName() + "-" + BUCKET_NAME; + StorageReaderImpl storageReader = new StorageReaderImpl(tenantInfo, null, + true); + String resultBucketName = storageReader.getTenantBucketName(); + assertEquals(bucketName, resultBucketName); + } + + @Test + public void should_returnBucketName_when_IsFullBucketName_is_false() { + when(tenantInfo.getName()).thenReturn("tenant1"); + when(tenantInfo.getProjectId()).thenReturn("projectId1"); + String bucketName = tenantInfo.getName() + "-" + BUCKET_NAME; + StorageReaderImpl storageReader = new StorageReaderImpl(tenantInfo, null, + false); + String resultBucketName = storageReader.getTenantBucketName(); + assertEquals(bucketName, resultBucketName); + } + + @Test + public void should_returnBucketName_when_IsFullBucketName_is_null() { + when(tenantInfo.getName()).thenReturn("tenant1"); + when(tenantInfo.getProjectId()).thenReturn("projectId1"); + String bucketName = tenantInfo.getName() + "-" + BUCKET_NAME; + StorageReaderImpl storageReader = new StorageReaderImpl(tenantInfo, null, + null); + String resultBucketName = storageReader.getTenantBucketName(); + assertEquals(bucketName, resultBucketName); + } } \ No newline at end of file diff --git a/provider/legal-gcp/src/test/resources/application.properties b/provider/legal-gcp/src/test/resources/application.properties new file mode 100644 index 0000000000000000000000000000000000000000..f8c687e1ba7be85f17ef24de4ce700caf3429683 --- /dev/null +++ b/provider/legal-gcp/src/test/resources/application.properties @@ -0,0 +1,3 @@ +enable.full.bucket.name=false + + diff --git a/provider/legal-ibm/pom.xml b/provider/legal-ibm/pom.xml index 18673e544a706354376c7ce36c8006183c176813..1ba91705c3691d33684f0717b158598478b8263e 100644 --- a/provider/legal-ibm/pom.xml +++ b/provider/legal-ibm/pom.xml @@ -13,7 +13,7 @@ <artifactId>legal-ibm</artifactId> <packaging>jar</packaging> <properties> - <os-core-lib-ibm.version>0.0.18</os-core-lib-ibm.version> + <os-core-lib-ibm.version>0.3.6-SNAPSHOT</os-core-lib-ibm.version> </properties> @@ -87,7 +87,7 @@ <version>2.5.8</version> <type>pom</type> </dependency> - + <dependency> <groupId>org.springframework.boot</groupId> <artifactId>spring-boot-starter-test</artifactId> diff --git a/testing/legal-test-aws/build-aws/prepare-dist.sh b/testing/legal-test-aws/build-aws/prepare-dist.sh new file mode 100755 index 0000000000000000000000000000000000000000..0366b1b7b34088093c75e2030528754384a3609b --- /dev/null +++ b/testing/legal-test-aws/build-aws/prepare-dist.sh @@ -0,0 +1,31 @@ +# This script prepares the dist directory for the integration tests. +# Must be run from the root of the repostiory + +set -e + +OUTPUT_DIR="${OUTPUT_DIR:-dist}" + +INTEGRATION_TEST_OUTPUT_DIR=${INTEGRATION_TEST_OUTPUT_DIR:-$OUTPUT_DIR}/testing/integration +INTEGRATION_TEST_OUTPUT_BIN_DIR=${INTEGRATION_TEST_OUTPUT_DIR:-$INTEGRATION_TEST_OUTPUT_DIR}/bin +INTEGRATION_TEST_SOURCE_DIR=testing +INTEGRATION_TEST_SOURCE_DIR_AWS="$INTEGRATION_TEST_SOURCE_DIR"/legal-test-aws +INTEGRATION_TEST_SOURCE_DIR_CORE="$INTEGRATION_TEST_SOURCE_DIR"/legal-test-core +echo "--Source directories variables--" +echo $INTEGRATION_TEST_SOURCE_DIR_AWS +echo $INTEGRATION_TEST_SOURCE_DIR_CORE +echo "--Output directories variables--" +echo $OUTPUT_DIR +echo $INTEGRATION_TEST_OUTPUT_DIR +echo $INTEGRATION_TEST_OUTPUT_BIN_DIR + +rm -rf "$INTEGRATION_TEST_OUTPUT_DIR" +mkdir -p "$INTEGRATION_TEST_OUTPUT_DIR" && mkdir -p "$INTEGRATION_TEST_OUTPUT_BIN_DIR" +echo "Building integration testing assemblies and gathering artifacts..." +mvn install -f "$INTEGRATION_TEST_SOURCE_DIR_CORE"/pom.xml +mvn install dependency:copy-dependencies -DskipTests -f "$INTEGRATION_TEST_SOURCE_DIR_AWS"/pom.xml -DincludeGroupIds=org.opengroup.osdu -Dmdep.copyPom +cp "$INTEGRATION_TEST_SOURCE_DIR_AWS"/target/dependency/* "${INTEGRATION_TEST_OUTPUT_BIN_DIR}" +(cd "${INTEGRATION_TEST_OUTPUT_BIN_DIR}" && ls *.jar | sed -e 's/\.jar$//' | xargs -I {} echo mvn install:install-file -Dfile={}.jar -DpomFile={}.pom >> install-deps.sh) +chmod +x "${INTEGRATION_TEST_OUTPUT_BIN_DIR}"/install-deps.sh +mvn clean -f "$INTEGRATION_TEST_SOURCE_DIR_AWS"/pom.xml +cp -R "$INTEGRATION_TEST_SOURCE_DIR_AWS"/* "${INTEGRATION_TEST_OUTPUT_DIR}"/ + diff --git a/testing/legal-test-aws/build-aws/run-tests.sh b/testing/legal-test-aws/build-aws/run-tests.sh new file mode 100755 index 0000000000000000000000000000000000000000..d5c64f7d2e39297671f79f0fb1f5cf6eae68e85d --- /dev/null +++ b/testing/legal-test-aws/build-aws/run-tests.sh @@ -0,0 +1,56 @@ +# This script executes the test and copies reports to the provided output directory +# To call this script from the service working directory +# ./dist/testing/integration/build-aws/run-tests.sh "./reports/" + + +SCRIPT_SOURCE_DIR=$(dirname "$0") +echo "Script source location" +echo "$SCRIPT_SOURCE_DIR" +(cd "$SCRIPT_SOURCE_DIR"/../bin && ./install-deps.sh) + +#### ADD REQUIRED ENVIRONMENT VARIABLES HERE ############################################### +# The following variables are automatically populated from the environment during integration testing +# see os-deploy-aws/build-aws/integration-test-env-variables.py for an updated list + +### DYNAMIC PARMETERS ### +# AWS_COGNITO_CLIENT_ID +# ELASTIC_HOST +# ELASTIC_PORT +# FILE_URL +# LEGAL_URL +# RESOURCE_PREFIX +# SEARCH_URL +# LEGAL_QUEUE +# LEGAL_S3_BUCKET + +### STATIC PARAMETERS ###: KEEP IN ALPHABETICAL ORDER +# ** DO NOT ADD VARIABLES NOT USED BY THIS SERVICE!!!! ** +export AWS_COGNITO_AUTH_FLOW=USER_PASSWORD_AUTH +export AWS_COGNITO_AUTH_PARAMS_PASSWORD=$ADMIN_PASSWORD +export AWS_COGNITO_AUTH_PARAMS_USER=$ADMIN_USER +export AWS_COGNITO_CLIENT_ID=$AWS_COGNITO_CLIENT_ID +export AWS_S3_ENDPOINT=s3.us-east-1.amazonaws.com +export AWS_S3_REGION=us-east-1 +export DYNAMO_DB_ENDPOINT=dynamodb.us-east-1.amazonaws.com +export DYNAMO_DB_REGION=us-east-1 +export HOST_URL=$LEGAL_URL +export MY_TENANT=int-test-legal +export S3_LEGAL_CONFIG_BUCKET=$LEGAL_S3_BUCKET +export SKIP_HTTP_TESTS=true +export TABLE_PREFIX=$RESOURCE_PREFIX + + +#### RUN INTEGRATION TEST ######################################################################### + +mvn test -f "$SCRIPT_SOURCE_DIR"/../pom.xml +TEST_EXIT_CODE=$? + +#### COPY TEST REPORTS ######################################################################### + +if [ -n "$1" ] + then + mkdir -p "$1" + cp -R "$SCRIPT_SOURCE_DIR"/../target/surefire-reports "$1" +fi + +exit $TEST_EXIT_CODE \ No newline at end of file diff --git a/testing/legal-test-aws/pom.xml b/testing/legal-test-aws/pom.xml index d9663e38f42ec914b4f1b0eddde0c62fb6360656..9fe713b0c882d0dd4d69c89ff164691ddf890b30 100644 --- a/testing/legal-test-aws/pom.xml +++ b/testing/legal-test-aws/pom.xml @@ -1,6 +1,6 @@ <?xml version="1.0" encoding="UTF-8"?> <!-- - Copyright © Amazon Web Services + Copyright © 2020 Amazon Web Services Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -40,7 +40,7 @@ <dependency> <groupId>org.opengroup.osdu.core.aws</groupId> <artifactId>os-core-lib-aws</artifactId> - <version>0.2.0</version> + <version>0.3.7</version> </dependency> <!-- AWS managed packages --> diff --git a/testing/legal-test-azure/pom.xml b/testing/legal-test-azure/pom.xml index 042cc6961e7b0297d068a9b855347a9e6c107364..6819637aae801d1151c9ece4fc353d403fedac4f 100644 --- a/testing/legal-test-azure/pom.xml +++ b/testing/legal-test-azure/pom.xml @@ -30,6 +30,8 @@ <maven.compiler.source>1.8</maven.compiler.source> <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding> <project.main.basedir>${project.basedir}</project.main.basedir> + <jackson.version>2.10.1</jackson.version> + <corelib.version>0.0.28</corelib.version> </properties> <dependencies> <dependency> @@ -41,13 +43,13 @@ <dependency> <groupId>com.azure</groupId> <artifactId>azure-storage-blob</artifactId> - <version>12.0.0-preview.2</version> + <version>12.3.0</version> </dependency> <!-- https://mvnrepository.com/artifact/com.microsoft.azure/azure-servicebus --> <dependency> <groupId>com.microsoft.azure</groupId> <artifactId>azure-servicebus</artifactId> - <version>1.2.8</version> + <version>3.1.4</version> </dependency> <!-- Tests --> @@ -56,7 +58,31 @@ <artifactId>junit</artifactId> <version>4.12</version> <scope>test</scope> - </dependency> + </dependency> + <!-- Added depdencies Azure Service Principle --> + <dependency> + <groupId>io.netty</groupId> + <artifactId>netty-all</artifactId> + <version>4.1.50.Final</version> + </dependency> + <dependency> + <groupId>org.opengroup.osdu</groupId> + <artifactId>core-lib-azure</artifactId> + <version>${corelib.version}</version> + <scope>test</scope> + <exclusions> + <exclusion> + <groupId>com.azure</groupId> + <artifactId>azure-core-http-netty</artifactId> + </exclusion> + </exclusions> + </dependency> + <dependency> + <groupId>com.fasterxml.jackson.core</groupId> + <artifactId>jackson-databind</artifactId> + <version>${jackson.version}</version> + <scope>test</scope> + </dependency> </dependencies> <repositories> diff --git a/testing/legal-test-azure/src/test/java/org/opengroup/osdu/legal/util/AzureLegalTagUtils.java b/testing/legal-test-azure/src/test/java/org/opengroup/osdu/legal/util/AzureLegalTagUtils.java index 54cef5b34107d630e9c28e22c1fcfb426b2ad218..f21d324fe88691eb624e64c0dae4526760f7611b 100644 --- a/testing/legal-test-azure/src/test/java/org/opengroup/osdu/legal/util/AzureLegalTagUtils.java +++ b/testing/legal-test-azure/src/test/java/org/opengroup/osdu/legal/util/AzureLegalTagUtils.java @@ -16,50 +16,90 @@ package org.opengroup.osdu.legal.util; import java.io.ByteArrayInputStream; import java.io.IOException; -import java.io.InputStream; -import java.nio.charset.StandardCharsets; -import com.azure.storage.blob.BlobServiceClient; -import com.azure.storage.blob.BlobServiceClientBuilder; -import com.azure.storage.blob.BlockBlobClient; -import com.azure.storage.blob.ContainerClient; -import com.azure.storage.common.credentials.SharedKeyCredential; +import com.azure.identity.ClientSecretCredential; +import com.azure.identity.ClientSecretCredentialBuilder; +import com.azure.storage.blob.BlobContainerClient; +import com.azure.storage.blob.BlobContainerClientBuilder; +import com.azure.storage.blob.BlobUrlParts; +import com.azure.storage.blob.specialized.BlockBlobClient; import com.google.common.base.Strings; +import org.opengroup.osdu.azure.util.AzureServicePrincipal; public class AzureLegalTagUtils extends LegalTagUtils { private static final String FILE_NAME = "Legal_COO.json"; private static final String CONTAINER_NAME_AZURE = "legal-service-azure-configuration"; + private static String clientSecret = System.getProperty("AZURE_TESTER_SERVICEPRINCIPAL_SECRET", System.getenv("AZURE_TESTER_SERVICEPRINCIPAL_SECRET")); + private static String clientId = System.getProperty("INTEGRATION_TESTER", System.getenv("INTEGRATION_TESTER")); + private static String tenantId = System.getProperty("AZURE_AD_TENANT_ID", System.getenv("AZURE_AD_TENANT_ID")); + private static String storageAccount = System.getProperty("AZURE_LEGAL_STORAGE_ACCOUNT", System.getenv("AZURE_LEGAL_STORAGE_ACCOUNT")).toLowerCase(); + private static String app_resource_id = System.getProperty("AZURE_AD_APP_RESOURCE_ID", System.getenv("AZURE_AD_APP_RESOURCE_ID")); @Override public synchronized void uploadTenantTestingConfigFile() { try { - String storageAccount = System.getProperty("AZURE_LEGAL_STORAGE_ACCOUNT", System.getenv("AZURE_LEGAL_STORAGE_ACCOUNT")).toLowerCase(); - String storageAccountKey = System.getProperty("AZURE_LEGAL_STORAGE_KEY", System.getenv("AZURE_LEGAL_STORAGE_KEY")); - SharedKeyCredential credential = new SharedKeyCredential(storageAccount, storageAccountKey); - BlobServiceClient storageClient = new BlobServiceClientBuilder() - .endpoint(String.format("https://%s.blob.core.windows.net", storageAccount)) - .credential(credential) - .buildClient(); - ContainerClient containerClient = storageClient - .getContainerClient(CONTAINER_NAME_AZURE); - BlockBlobClient blobClient = containerClient.getBlockBlobClient(FILE_NAME); - String content = readTestFile("TenantConfigTestingPurpose.json"); - InputStream dataStream = new ByteArrayInputStream(content.getBytes(StandardCharsets.UTF_8)); - blobClient.upload(dataStream, content.length()); - dataStream.close(); + String blobPath = generateBlobPath(storageAccount, CONTAINER_NAME_AZURE, FILE_NAME); + BlobUrlParts parts = BlobUrlParts.parse(blobPath); + BlobContainerClient blobContainerClient = getBlobContainerClient(parts.getAccountName(), parts.getBlobContainerName()); + if (!blobContainerClient.exists()) { + createContainer(parts.getBlobContainerName()); + } + BlockBlobClient blockBlobClient = blobContainerClient.getBlobClient(parts.getBlobName()).getBlockBlobClient(); + if (!blockBlobClient.exists()) { + String content = readTestFile("TenantConfigTestingPurpose.json"); + try (ByteArrayInputStream dataStream = new ByteArrayInputStream(content.getBytes())) { + blockBlobClient.upload(dataStream, content.length()); + } catch (Exception e) { + e.printStackTrace(); + throw new AssertionError(String.format("Error: Could not create test %s file blob", parts.getBlobName()), e); + } + } }catch (IOException ex){ ex.printStackTrace(); } } - + + private static String generateContainerPath(String accountName, String containerName) { + return String.format("https://%s.blob.core.windows.net/%s", accountName, containerName); + } + + public void createContainer(String containerName) + { + String containerPath = generateContainerPath(storageAccount, containerName); + BlobUrlParts parts = BlobUrlParts.parse(containerPath); + BlobContainerClient blobContainerClient = getBlobContainerClient(parts.getAccountName(), parts.getBlobContainerName()); + if(!blobContainerClient.exists()){ + blobContainerClient.create(); + + } + } + + private static String generateBlobPath(String accountName, String containerName, String blobName) { + return String.format("https://%s.blob.core.windows.net/%s/%s", accountName, containerName, blobName); + } + + private BlobContainerClient getBlobContainerClient(String accountName, String containerName) { + ClientSecretCredential clientSecretCredential = new ClientSecretCredentialBuilder() + .clientSecret(clientSecret) + .clientId(clientId) + .tenantId(tenantId) + .build(); + BlobContainerClient blobContainerClient = new BlobContainerClientBuilder() + .endpoint(getBlobAccountUrl(accountName)) + .credential(clientSecretCredential) + .containerName(containerName) + .buildClient(); + return blobContainerClient; + } + + private static String getBlobAccountUrl(String accountName) { + return String.format("https://%s.blob.core.windows.net", accountName); + } + @Override public synchronized String accessToken() throws Exception { if (Strings.isNullOrEmpty(token)) { - String sp_id = System.getProperty("INTEGRATION_TESTER", System.getenv("INTEGRATION_TESTER")); - String sp_secret = System.getProperty("AZURE_TESTER_SERVICEPRINCIPAL_SECRET", System.getenv("AZURE_TESTER_SERVICEPRINCIPAL_SECRET")); - String tenant_id = System.getProperty("AZURE_AD_TENANT_ID", System.getenv("AZURE_AD_TENANT_ID")); - String app_resource_id = System.getProperty("AZURE_AD_APP_RESOURCE_ID", System.getenv("AZURE_AD_APP_RESOURCE_ID")); - token = AzureServicePrincipal.getIdToken(sp_id, sp_secret, tenant_id, app_resource_id); + token = new AzureServicePrincipal().getIdToken(clientId, clientSecret, tenantId, app_resource_id); } return "Bearer " + token; } diff --git a/testing/legal-test-azure/src/test/java/org/opengroup/osdu/legal/util/AzureServicePrincipal.java b/testing/legal-test-azure/src/test/java/org/opengroup/osdu/legal/util/AzureServicePrincipal.java deleted file mode 100644 index 84b9a699a3a09ae6982ad788ae84baa6c6b11a39..0000000000000000000000000000000000000000 --- a/testing/legal-test-azure/src/test/java/org/opengroup/osdu/legal/util/AzureServicePrincipal.java +++ /dev/null @@ -1,83 +0,0 @@ -// Copyright © Microsoft Corporation -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package org.opengroup.osdu.legal.util; - -import com.google.gson.Gson; -import com.google.gson.JsonObject; - -import java.io.BufferedReader; -import java.io.DataOutputStream; -import java.io.InputStreamReader; -import java.io.UnsupportedEncodingException; -import java.net.HttpURLConnection; -import java.net.URL; -import java.net.URLEncoder; -import java.util.HashMap; -import java.util.Map; - -public class AzureServicePrincipal { - public static String getIdToken(String sp_id, String sp_secret, String tenant_id, String app_resource_id) throws Exception { - String aad_endpoint = String.format("https://login.microsoftonline.com/%s/oauth2/token", tenant_id); - URL url = new URL(aad_endpoint); - HttpURLConnection con = (HttpURLConnection) url.openConnection(); - con.setRequestMethod("POST"); - con.setRequestProperty("Content-Type", "application/x-www-form-urlencoded"); - - Map<String, String> parameters = new HashMap<>(); - parameters.put("grant_type", "client_credentials"); - parameters.put("client_id", sp_id); - parameters.put("client_secret", sp_secret); - parameters.put("resource", app_resource_id); - - con.setDoOutput(true); - DataOutputStream out = new DataOutputStream(con.getOutputStream()); - out.writeBytes(getParamsString(parameters)); - out.flush(); - out.close(); - - BufferedReader in = new BufferedReader( - new InputStreamReader(con.getInputStream())); - String inputLine; - StringBuffer content = new StringBuffer(); - while ((inputLine = in.readLine()) != null) { - content.append(inputLine); - } - in.close(); - - con.disconnect(); - - Gson gson = new Gson(); - JsonObject jobj = gson.fromJson(content.toString(), JsonObject.class); - String token = jobj.get("access_token").getAsString(); - return token; - } - - private static String getParamsString(Map<String, String> params) - throws UnsupportedEncodingException { - StringBuilder result = new StringBuilder(); - - for (Map.Entry<String, String> entry : params.entrySet()) { - result.append(URLEncoder.encode(entry.getKey(), "UTF-8")); - result.append("="); - result.append(URLEncoder.encode(entry.getValue(), "UTF-8")); - result.append("&"); - } - - String resultString = result.toString(); - return resultString.length() > 0 - ? resultString.substring(0, resultString.length() - 1) - : resultString; - } -} diff --git a/testing/legal-test-gcp/pom.xml b/testing/legal-test-gcp/pom.xml index 03535c75e84ed733aa2e97995864ec9df7e2af76..a8c42dc0b3a6f4f93a8aeeac551536b1e3b43681 100644 --- a/testing/legal-test-gcp/pom.xml +++ b/testing/legal-test-gcp/pom.xml @@ -117,6 +117,12 @@ <artifactId>guava</artifactId> <version>27.1-jre</version> </dependency> + <dependency> + <groupId>org.projectlombok</groupId> + <artifactId>lombok</artifactId> + <version>1.18.2</version> + <scope>provided</scope> + </dependency> <!-- Tests --> <dependency> @@ -124,9 +130,22 @@ <artifactId>junit</artifactId> <version>4.12</version> <scope>test</scope> - </dependency> + </dependency> + + <!--Logging--> + <dependency> + <groupId>org.apache.logging.log4j</groupId> + <artifactId>log4j-to-slf4j</artifactId> + <version>2.11.2</version> + </dependency> + <dependency> + <groupId>org.slf4j</groupId> + <artifactId>slf4j-jdk14</artifactId> + <version>1.8.0-beta4</version> + </dependency> </dependencies> + <!-- can remove this? --> <repositories> <repository> diff --git a/testing/legal-test-gcp/src/test/java/org/opengroup/osdu/legal/util/GCPLegalTagUtils.java b/testing/legal-test-gcp/src/test/java/org/opengroup/osdu/legal/util/GCPLegalTagUtils.java index 59cc43ce3dee1321ecadde7b3fbf24aebb151e22..af9a44b8410492abb8c814af80b631325050d91f 100644 --- a/testing/legal-test-gcp/src/test/java/org/opengroup/osdu/legal/util/GCPLegalTagUtils.java +++ b/testing/legal-test-gcp/src/test/java/org/opengroup/osdu/legal/util/GCPLegalTagUtils.java @@ -12,7 +12,13 @@ import com.google.cloud.storage.BlobInfo; import com.google.cloud.storage.Storage; import com.google.cloud.storage.StorageOptions; import com.google.common.collect.Lists; +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.util.Base64; +import lombok.extern.java.Log; +@Log public class GCPLegalTagUtils extends LegalTagUtils { private static final String BUCKET_NAME = "legal-service-configuration"; private static final String FILE_NAME = "Legal_COO.json"; @@ -38,12 +44,30 @@ public class GCPLegalTagUtils extends LegalTagUtils { } } - private static String getTenantBucketName() { - String tenantName = System.getProperty("MY_TENANT_PROJECT", System.getenv("MY_TENANT_PROJECT")).toLowerCase(); - String projectName = System.getProperty("GCLOUD_PROJECT", System.getenv("GCLOUD_PROJECT")).toLowerCase(); - return projectName + "-" + tenantName + "-" + BUCKET_NAME; + private static String getTenantBucketName() { + String tenantName = System + .getProperty("MY_TENANT_PROJECT", System.getenv("MY_TENANT_PROJECT")).toLowerCase(); + String projectName = System.getProperty("GCLOUD_PROJECT", System.getenv("GCLOUD_PROJECT")) + .toLowerCase(); + String enableFullBucketName = System.getProperty("ENABLE_FULL_BUCKET_NAME", + System.getenv("ENABLE_FULL_BUCKET_NAME")); + + enableFullBucketName = (Strings.isNullOrEmpty(enableFullBucketName) ? "false" + : enableFullBucketName).toLowerCase(); + + log.info("ENABLE_FULL_BUCKET_NAME = " + enableFullBucketName); + + String bucketName; + if (Boolean.parseBoolean(enableFullBucketName)) { + bucketName = projectName + "-" + tenantName + "-" + BUCKET_NAME; + } else { + bucketName = tenantName + "-" + BUCKET_NAME; } + log.info("bucketName = " + bucketName); + return bucketName; + } + @Override public synchronized String accessToken() throws Exception { if (Strings.isNullOrEmpty(token)) {