From 04d426cdbc6e670c8cb2f7544724777455909c38 Mon Sep 17 00:00:00 2001 From: Stanislav Riabokon Date: Fri, 20 May 2022 17:45:30 +0400 Subject: [PATCH 1/6] Int tests for Anthos. --- provider/schema-gcp/README.md | 34 +------ provider/schema-gcp/docs/anthos/README.md | 33 +++++++ provider/schema-gcp/docs/gcp/README.md | 31 ++++++ testing/pom.xml | 1 + testing/schema-test-anthos/pom.xml | 20 ++++ .../src/test/resources/logback-test.xml | 13 +++ .../opengroup/osdu/schema/util/AuthUtil.java | 5 +- .../schema/util/GoogleServiceAccount.java | 81 ---------------- .../schema/util/gcp/GoogleServiceAccount.java | 96 +++++++++++++++++++ .../schema/util/gcp/OpenIDTokenProvider.java | 80 ++++++++++++++++ .../util/gcp/config/OpenIDProviderConfig.java | 76 +++++++++++++++ 11 files changed, 358 insertions(+), 112 deletions(-) create mode 100644 testing/schema-test-anthos/pom.xml create mode 100644 testing/schema-test-anthos/src/test/resources/logback-test.xml delete mode 100644 testing/schema-test-core/src/test/java/org/opengroup/osdu/schema/util/GoogleServiceAccount.java create mode 100644 testing/schema-test-core/src/test/java/org/opengroup/osdu/schema/util/gcp/GoogleServiceAccount.java create mode 100644 testing/schema-test-core/src/test/java/org/opengroup/osdu/schema/util/gcp/OpenIDTokenProvider.java create mode 100644 testing/schema-test-core/src/test/java/org/opengroup/osdu/schema/util/gcp/config/OpenIDProviderConfig.java diff --git a/provider/schema-gcp/README.md b/provider/schema-gcp/README.md index 7c6a96cd..ce9373d5 100644 --- a/provider/schema-gcp/README.md +++ b/provider/schema-gcp/README.md @@ -94,36 +94,10 @@ Once the above Prerequisite are done, we can follow the below steps to run the s You can access the service APIs by following the service contract in [schema.yaml](docs/api/schema.yaml) ## Testing - - ### Running E2E Tests - This section describes how to run cloud OSDU E2E tests (testing/schema-test-core). - - You will need to have the following environment variables defined. - - | name | value | description | sensitive? | source | - | --- | --- | --- | --- | --- | - | `INTEGRATION_TEST_AUDIENCE` | `*****.apps.googleusercontent.com` | client application ID | yes | https://console.cloud.google.com/apis/credentials | - | `VENDOR` | `gcp` | Use value 'gcp' to run gcp tests | no | - | - | `HOST` | ex`http://localhost:8080` | Schema service host | no | - | - | `INTEGRATION_TESTER` | `********` | Service account base64 encoded string for API calls. Note: this user must have entitlements configured already | yes | https://console.cloud.google.com/iam-admin/serviceaccounts | - | `PRIVATE_TENANT2` | ex`opendes` | OSDU tenant used for testing | no | - | - | `PRIVATE_TENANT1` | ex`osdu` | OSDU tenant used for testing | no | - | - | `SHARED_TENANT` | ex`common` | OSDU tenant used for testing | no | - | - - **Entitlements configuration for integration accounts** - - | INTEGRATION_TESTER | - | --- | - | users
service.schema-service.system-admin
service.entitlements.user
service.schema-service.viewers
service.schema-service.editors
data.integration.test
data.test1 | - - Execute following command to build code and run all the integration tests: - - ```bash - # Note: this assumes that the environment variables for integration tests as outlined - # above are already exported in your environment. - # build + install integration test core - $ (cd testing/schema-test-core/ && mvn clean test) - ``` +#### Anthos: +[Anthos Testing](docs/anthos/README.md) +#### GCP: +[Gcp Testing](docs/gcp/README.md) ## Deployment diff --git a/provider/schema-gcp/docs/anthos/README.md b/provider/schema-gcp/docs/anthos/README.md index 0700bc6c..0f470328 100644 --- a/provider/schema-gcp/docs/anthos/README.md +++ b/provider/schema-gcp/docs/anthos/README.md @@ -39,6 +39,39 @@ and usage in mixed mode was not tested. Usage of spring profiles is preferred. | `OBMDRIVER` | `gcs` or `minio` | Obm driver mode that defines which object storage will be used | no | - | | `SERVICE_TOKEN_PROVIDER` | `GCP` or `OPENID` |Service account token provider, `GCP` means use Google service account `OPEIND` means use OpenId provider like `Keycloak` | no | - | +## Testing +### Running E2E Tests +This section describes how to run cloud OSDU E2E tests (testing/schema-test-core). + +You will need to have the following environment variables defined. + +| name | value | description | sensitive? | source | + | --- | --- | --- | --- | --- | +| `VENDOR` | `gcp` | Use value 'gcp' to run gcp tests | no | - | +| `HOST` | ex`http://localhost:8080` | Schema service host | no | - | +| `PRIVATE_TENANT2` | ex`opendes` | OSDU tenant used for testing | no | - | +| `PRIVATE_TENANT1` | ex`osdu` | OSDU tenant used for testing | no | - | +| `SHARED_TENANT` | ex`common` | OSDU tenant used for testing | no | - | +| `TEST_OPENID_PROVIDER_CLIENT_ID` | `********` | Client Id for `$INTEGRATION_TESTER` | yes | -- | +| `TEST_OPENID_PROVIDER_CLIENT_SECRET` | `********` | | Client secret for `$INTEGRATION_TESTER` | -- | +| `INTEGRATION_TESTER_EMAIL` | `datafier@service.local` | | Email of `$INTEGRATION_TESTER` | -- | +| `TEST_OPENID_PROVIDER_URL` | `https://keycloak.com/auth/realms/osdu` | OpenID provider url | yes | -- | + +**Entitlements configuration for integration accounts** + +| INTEGRATION_TESTER | + | --- | +| users
service.schema-service.system-admin
service.entitlements.user
service.schema-service.viewers
service.schema-service.editors
data.integration.test
data.test1 | + +Execute following command to build code and run all the integration tests: + + ```bash + # Note: this assumes that the environment variables for integration tests as outlined + # above are already exported in your environment. + # build + install integration test core + $ (cd testing/schema-test-core/ && mvn clean test) + ``` + ### Properties set in Partition service: Note that properties can be set in Partition as `sensitive` in that case in property `value` should be present not value itself, but ENV variable name. diff --git a/provider/schema-gcp/docs/gcp/README.md b/provider/schema-gcp/docs/gcp/README.md index d2bc7deb..589d06fb 100644 --- a/provider/schema-gcp/docs/gcp/README.md +++ b/provider/schema-gcp/docs/gcp/README.md @@ -36,6 +36,37 @@ and usage in mixed mode was not tested. Usage of spring profiles is preferred. | `OBMDRIVER` | `gcs` or `minio` | Obm driver mode that defines which object storage will be used | no | - | | `SERVICE_TOKEN_PROVIDER` | `GCP` or `OPENID` |Service account token provider, `GCP` means use Google service account `OPEIND` means use OpenId provider like `Keycloak` | no | - | +## Testing +### Running E2E Tests +This section describes how to run cloud OSDU E2E tests (testing/schema-test-core). + +You will need to have the following environment variables defined. + +| name | value | description | sensitive? | source | + | --- | --- | --- | --- | --- | +| `INTEGRATION_TEST_AUDIENCE` | `*****.apps.googleusercontent.com` | client application ID | yes | https://console.cloud.google.com/apis/credentials | +| `VENDOR` | `gcp` | Use value 'gcp' to run gcp tests | no | - | +| `HOST` | ex`http://localhost:8080` | Schema service host | no | - | +| `INTEGRATION_TESTER` | `********` | Service account base64 encoded string for API calls. Note: this user must have entitlements configured already | yes | https://console.cloud.google.com/iam-admin/serviceaccounts | +| `PRIVATE_TENANT2` | ex`opendes` | OSDU tenant used for testing | no | - | +| `PRIVATE_TENANT1` | ex`osdu` | OSDU tenant used for testing | no | - | +| `SHARED_TENANT` | ex`common` | OSDU tenant used for testing | no | - | + +**Entitlements configuration for integration accounts** + +| INTEGRATION_TESTER | + | --- | +| users
service.schema-service.system-admin
service.entitlements.user
service.schema-service.viewers
service.schema-service.editors
data.integration.test
data.test1 | + +Execute following command to build code and run all the integration tests: + + ```bash + # Note: this assumes that the environment variables for integration tests as outlined + # above are already exported in your environment. + # build + install integration test core + $ (cd testing/schema-test-core/ && mvn clean test) + ``` + ## Datastore configuration: There must be a namespace `dataecosystem`. diff --git a/testing/pom.xml b/testing/pom.xml index 340eb7a1..8a49e81f 100644 --- a/testing/pom.xml +++ b/testing/pom.xml @@ -19,6 +19,7 @@ schema-test-core schema-test-gcp schema-test-azure + schema-test-anthos diff --git a/testing/schema-test-anthos/pom.xml b/testing/schema-test-anthos/pom.xml new file mode 100644 index 00000000..55aadd93 --- /dev/null +++ b/testing/schema-test-anthos/pom.xml @@ -0,0 +1,20 @@ + + 4.0.0 + + + org.opengroup.osdu + os-schema-test + 0.15.0-SNAPSHOT + ../pom.xml + + + org.opengroup.osdu + os-schema-test-anthos + 0.15.0-SNAPSHOT + jar + + + schema-test-gcp + Anthos implementation of test project for schema service + + \ No newline at end of file diff --git a/testing/schema-test-anthos/src/test/resources/logback-test.xml b/testing/schema-test-anthos/src/test/resources/logback-test.xml new file mode 100644 index 00000000..22c6175d --- /dev/null +++ b/testing/schema-test-anthos/src/test/resources/logback-test.xml @@ -0,0 +1,13 @@ + + + + + + %yellow([%thread]) %highlight(| %-5level |) %green(%d) %cyan(| %logger{15} |) %highlight(%msg) %n + utf8 + + + + + + diff --git a/testing/schema-test-core/src/test/java/org/opengroup/osdu/schema/util/AuthUtil.java b/testing/schema-test-core/src/test/java/org/opengroup/osdu/schema/util/AuthUtil.java index 63aa951e..90e96af5 100644 --- a/testing/schema-test-core/src/test/java/org/opengroup/osdu/schema/util/AuthUtil.java +++ b/testing/schema-test-core/src/test/java/org/opengroup/osdu/schema/util/AuthUtil.java @@ -2,8 +2,9 @@ package org.opengroup.osdu.schema.util; import com.google.common.base.Strings; import org.opengroup.osdu.azure.util.AzureServicePrincipal; -import org.opengroup.osdu.core.aws.cognito.AWSCognitoClient; import org.opengroup.osdu.core.ibm.util.IdentityClient; +import org.opengroup.osdu.schema.util.gcp.GoogleServiceAccount; +import org.opengroup.osdu.schema.util.gcp.OpenIDTokenProvider; public class AuthUtil { @@ -25,6 +26,8 @@ public class AuthUtil { token = new AzureServicePrincipal().getIdToken(sp_id, sp_secret, tenant_id, app_resource_id); } else if (Strings.isNullOrEmpty(token) && vendor.equals("ibm")) { token = IdentityClient.getTokenForUserWithAccess(); + } else if (Strings.isNullOrEmpty(token) && vendor.equals("anthos")){ + token = new OpenIDTokenProvider().getToken(); } return "Bearer " + token; } diff --git a/testing/schema-test-core/src/test/java/org/opengroup/osdu/schema/util/GoogleServiceAccount.java b/testing/schema-test-core/src/test/java/org/opengroup/osdu/schema/util/GoogleServiceAccount.java deleted file mode 100644 index 772b8334..00000000 --- a/testing/schema-test-core/src/test/java/org/opengroup/osdu/schema/util/GoogleServiceAccount.java +++ /dev/null @@ -1,81 +0,0 @@ -package org.opengroup.osdu.schema.util; - -import java.io.ByteArrayInputStream; -import java.io.IOException; -import java.io.InputStream; -import java.util.ArrayList; -import java.util.Base64; -import java.util.HashMap; -import java.util.Map; - -import org.apache.http.HttpResponse; -import org.apache.http.NameValuePair; -import org.apache.http.client.HttpClient; -import org.apache.http.client.entity.UrlEncodedFormEntity; -import org.apache.http.client.methods.HttpPost; -import org.apache.http.impl.client.DefaultHttpClient; -import org.apache.http.message.BasicNameValuePair; -import org.apache.http.util.EntityUtils; - -import com.google.auth.oauth2.ServiceAccountCredentials; -import com.google.gson.JsonObject; -import com.google.gson.JsonParser; - -import io.jsonwebtoken.JwtBuilder; -import io.jsonwebtoken.Jwts; -import io.jsonwebtoken.SignatureAlgorithm; - -public class GoogleServiceAccount { - - final ServiceAccountCredentials serviceAccount; - - public GoogleServiceAccount(String serviceAccountEncoded) throws IOException { - this(Base64.getDecoder().decode(serviceAccountEncoded)); - } - - public GoogleServiceAccount(byte[] serviceAccountJson) throws IOException { - try (InputStream inputStream = new ByteArrayInputStream(serviceAccountJson)) { - - this.serviceAccount = ServiceAccountCredentials.fromStream(inputStream); - } - } - - public String getEmail() { - return this.serviceAccount.getClientEmail(); - } - - public String getAuthToken(String audience) throws IOException { - JwtBuilder jwtBuilder = Jwts.builder(); - - Map header = new HashMap<>(); - header.put("type", "JWT"); - header.put("alg", "RS256"); - jwtBuilder.setHeader(header); - - Map claims = new HashMap<>(); - claims.put("target_audience", audience); - claims.put("exp", System.currentTimeMillis() / 1000 + 3600); - claims.put("iat", System.currentTimeMillis() / 1000); - claims.put("iss", this.getEmail()); - claims.put("aud", "https://www.googleapis.com/oauth2/v4/token"); - jwtBuilder.addClaims(claims); - - jwtBuilder.signWith(SignatureAlgorithm.RS256, this.serviceAccount.getPrivateKey()); - String jwt = jwtBuilder.compact(); - - HttpPost httpPost = new HttpPost("https://www.googleapis.com/oauth2/v4/token"); - - ArrayList postParameters = new ArrayList<>(); - postParameters.add(new BasicNameValuePair("grant_type", "urn:ietf:params:oauth:grant-type:jwt-bearer")); - postParameters.add(new BasicNameValuePair("assertion", jwt)); - - HttpClient client = new DefaultHttpClient(); - - httpPost.setEntity(new UrlEncodedFormEntity(postParameters, "UTF-8")); - httpPost.setHeader("Content-Type", "application/x-www-form-urlencoded"); - HttpResponse response = client.execute(httpPost); - String responseEntity = EntityUtils.toString(response.getEntity()); - JsonObject content = new JsonParser().parse(responseEntity).getAsJsonObject(); - return content.get("id_token").getAsString(); - } -} diff --git a/testing/schema-test-core/src/test/java/org/opengroup/osdu/schema/util/gcp/GoogleServiceAccount.java b/testing/schema-test-core/src/test/java/org/opengroup/osdu/schema/util/gcp/GoogleServiceAccount.java new file mode 100644 index 00000000..5d05ccbe --- /dev/null +++ b/testing/schema-test-core/src/test/java/org/opengroup/osdu/schema/util/gcp/GoogleServiceAccount.java @@ -0,0 +1,96 @@ +/* + * Copyright 2020-2022 Google LLC + * Copyright 2020-2022 EPAM Systems, Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.opengroup.osdu.schema.util.gcp; + +import com.google.auth.oauth2.ServiceAccountCredentials; +import com.google.gson.JsonObject; +import com.google.gson.JsonParser; +import io.jsonwebtoken.JwtBuilder; +import io.jsonwebtoken.Jwts; +import io.jsonwebtoken.SignatureAlgorithm; +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.util.ArrayList; +import java.util.Base64; +import java.util.HashMap; +import java.util.Map; +import org.apache.http.HttpResponse; +import org.apache.http.NameValuePair; +import org.apache.http.client.HttpClient; +import org.apache.http.client.entity.UrlEncodedFormEntity; +import org.apache.http.client.methods.HttpPost; +import org.apache.http.impl.client.DefaultHttpClient; +import org.apache.http.message.BasicNameValuePair; +import org.apache.http.util.EntityUtils; + +public class GoogleServiceAccount { + + final ServiceAccountCredentials serviceAccount; + + public GoogleServiceAccount(String serviceAccountEncoded) throws IOException { + this(Base64.getDecoder().decode(serviceAccountEncoded)); + } + + public GoogleServiceAccount(byte[] serviceAccountJson) throws IOException { + try (InputStream inputStream = new ByteArrayInputStream(serviceAccountJson)) { + + this.serviceAccount = ServiceAccountCredentials.fromStream(inputStream); + } + } + + public String getEmail() { + return this.serviceAccount.getClientEmail(); + } + + public String getAuthToken(String audience) throws IOException { + JwtBuilder jwtBuilder = Jwts.builder(); + + Map header = new HashMap<>(); + header.put("type", "JWT"); + header.put("alg", "RS256"); + jwtBuilder.setHeader(header); + + Map claims = new HashMap<>(); + claims.put("target_audience", audience); + claims.put("exp", System.currentTimeMillis() / 1000 + 3600); + claims.put("iat", System.currentTimeMillis() / 1000); + claims.put("iss", this.getEmail()); + claims.put("aud", "https://www.googleapis.com/oauth2/v4/token"); + jwtBuilder.addClaims(claims); + + jwtBuilder.signWith(SignatureAlgorithm.RS256, this.serviceAccount.getPrivateKey()); + String jwt = jwtBuilder.compact(); + + HttpPost httpPost = new HttpPost("https://www.googleapis.com/oauth2/v4/token"); + + ArrayList postParameters = new ArrayList<>(); + postParameters.add( + new BasicNameValuePair("grant_type", "urn:ietf:params:oauth:grant-type:jwt-bearer")); + postParameters.add(new BasicNameValuePair("assertion", jwt)); + + HttpClient client = new DefaultHttpClient(); + + httpPost.setEntity(new UrlEncodedFormEntity(postParameters, "UTF-8")); + httpPost.setHeader("Content-Type", "application/x-www-form-urlencoded"); + HttpResponse response = client.execute(httpPost); + String responseEntity = EntityUtils.toString(response.getEntity()); + JsonObject content = new JsonParser().parse(responseEntity).getAsJsonObject(); + return content.get("id_token").getAsString(); + } +} diff --git a/testing/schema-test-core/src/test/java/org/opengroup/osdu/schema/util/gcp/OpenIDTokenProvider.java b/testing/schema-test-core/src/test/java/org/opengroup/osdu/schema/util/gcp/OpenIDTokenProvider.java new file mode 100644 index 00000000..c5f3cd22 --- /dev/null +++ b/testing/schema-test-core/src/test/java/org/opengroup/osdu/schema/util/gcp/OpenIDTokenProvider.java @@ -0,0 +1,80 @@ +/* + * Copyright 2020-2022 Google LLC + * Copyright 2020-2022 EPAM Systems, Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.opengroup.osdu.schema.util.gcp; + +import com.nimbusds.oauth2.sdk.AuthorizationGrant; +import com.nimbusds.oauth2.sdk.ClientCredentialsGrant; +import com.nimbusds.oauth2.sdk.ParseException; +import com.nimbusds.oauth2.sdk.Scope; +import com.nimbusds.oauth2.sdk.TokenRequest; +import com.nimbusds.oauth2.sdk.TokenResponse; +import com.nimbusds.oauth2.sdk.auth.ClientAuthentication; +import com.nimbusds.oauth2.sdk.auth.ClientSecretBasic; +import com.nimbusds.oauth2.sdk.auth.Secret; +import com.nimbusds.oauth2.sdk.id.ClientID; +import com.nimbusds.openid.connect.sdk.OIDCTokenResponseParser; +import java.io.IOException; +import java.net.URI; +import java.util.Objects; +import net.minidev.json.JSONObject; +import org.opengroup.osdu.schema.util.gcp.config.OpenIDProviderConfig; + +public class OpenIDTokenProvider { + + private static final OpenIDProviderConfig openIDProviderConfig = OpenIDProviderConfig.Instance(); + private static final String ID_TOKEN = "id_token"; + private final AuthorizationGrant clientGrant = new ClientCredentialsGrant(); + private final URI tokenEndpointURI; + private final Scope scope; + private final ClientAuthentication clientAuthentication; + + public OpenIDTokenProvider() { + this.tokenEndpointURI = openIDProviderConfig.getProviderMetadata().getTokenEndpointURI(); + this.scope = new Scope(openIDProviderConfig.getScopes()); + this.clientAuthentication = + new ClientSecretBasic( + new ClientID(openIDProviderConfig.getClientId()), + new Secret(openIDProviderConfig.getClientSecret()) + ); + } + + public String getToken() { + try { + return requestToken(); + } catch (ParseException | IOException e) { + throw new RuntimeException("Unable get credentials from INTEGRATION_TESTER variables", e); + } + } + + private String requestToken() throws ParseException, IOException { + TokenRequest request = new TokenRequest(this.tokenEndpointURI, this.clientAuthentication, + this.clientGrant, this.scope); + TokenResponse parse = OIDCTokenResponseParser.parse(request.toHTTPRequest().send()); + + if (!parse.indicatesSuccess()) { + throw new RuntimeException("Unable get credentials from INTEGRATION_TESTER variables"); + } + + JSONObject jsonObject = parse.toSuccessResponse().toJSONObject(); + String idTokenValue = jsonObject.getAsString(ID_TOKEN); + if (Objects.isNull(idTokenValue) || idTokenValue.isEmpty()) { + throw new RuntimeException("Unable get credentials from INTEGRATION_TESTER variables"); + } + return idTokenValue; + } +} \ No newline at end of file diff --git a/testing/schema-test-core/src/test/java/org/opengroup/osdu/schema/util/gcp/config/OpenIDProviderConfig.java b/testing/schema-test-core/src/test/java/org/opengroup/osdu/schema/util/gcp/config/OpenIDProviderConfig.java new file mode 100644 index 00000000..5eb76022 --- /dev/null +++ b/testing/schema-test-core/src/test/java/org/opengroup/osdu/schema/util/gcp/config/OpenIDProviderConfig.java @@ -0,0 +1,76 @@ +/* + * Copyright 2020-2022 Google LLC + * Copyright 2020-2022 EPAM Systems, Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.opengroup.osdu.schema.util.gcp.config; + +import com.nimbusds.oauth2.sdk.http.HTTPRequest; +import com.nimbusds.oauth2.sdk.http.HTTPResponse; +import com.nimbusds.oauth2.sdk.id.Issuer; +import com.nimbusds.openid.connect.sdk.op.OIDCProviderConfigurationRequest; +import com.nimbusds.openid.connect.sdk.op.OIDCProviderMetadata; + +public class OpenIDProviderConfig { + + private String clientId; + private String url; + private String clientSecret; + private String intTesterEmail; + private final String[] scopes = {"openid"}; + private static final OpenIDProviderConfig openIDProviderConfig = new OpenIDProviderConfig(); + private static OIDCProviderMetadata providerMetadata; + + public static OpenIDProviderConfig Instance() { + try { + openIDProviderConfig.clientId = System.getProperty("TEST_OPENID_PROVIDER_CLIENT_ID", + System.getenv("TEST_OPENID_PROVIDER_CLIENT_ID")); + openIDProviderConfig.url = System.getProperty("TEST_OPENID_PROVIDER_URL", + System.getenv("TEST_OPENID_PROVIDER_URL")); + openIDProviderConfig.clientSecret = System.getProperty("TEST_OPENID_PROVIDER_CLIENT_SECRET", + System.getenv("TEST_OPENID_PROVIDER_CLIENT_SECRET")); + openIDProviderConfig.intTesterEmail = System.getProperty("INTEGRATION_TESTER_EMAIL", + System.getenv("INTEGRATION_TESTER_EMAIL")); + Issuer issuer = new Issuer(openIDProviderConfig.url); + OIDCProviderConfigurationRequest request = new OIDCProviderConfigurationRequest(issuer); + HTTPRequest httpRequest = request.toHTTPRequest(); + HTTPResponse httpResponse = httpRequest.send(); + providerMetadata = OIDCProviderMetadata.parse(httpResponse.getContentAsJSONObject()); + } catch (Exception e) { + throw new RuntimeException("Malformed token provider configuration", e); + } + return openIDProviderConfig; + } + + public String getClientId() { + return clientId; + } + + public String getClientSecret() { + return clientSecret; + } + + public String[] getScopes() { + return scopes; + } + + public OIDCProviderMetadata getProviderMetadata() { + return providerMetadata; + } + + public String getIntTesterEmail() { + return intTesterEmail; + } +} \ No newline at end of file -- GitLab From f91792288f3f05cffea83d37ef2d8a4aa5901094 Mon Sep 17 00:00:00 2001 From: Mikhail Piatliou Date: Fri, 20 May 2022 18:52:00 +0400 Subject: [PATCH 2/6] GONRG-4968: updated pipeline to check anthos tests --- .gitlab-ci.yml | 1 + devops/gcp/pipeline/override-stages.yml | 13 ------------- 2 files changed, 1 insertion(+), 13 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index c570e48d..471ed375 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -62,6 +62,7 @@ include: file: "cloud-providers/azure.yml" - project: "osdu/platform/ci-cd-pipelines" + ref: "gcp-int-tests-anthos" file: "cloud-providers/osdu-gcp-gke.yml" - local: "devops/gcp/pipeline/override-stages.yml" diff --git a/devops/gcp/pipeline/override-stages.yml b/devops/gcp/pipeline/override-stages.yml index e60c7673..43324d61 100644 --- a/devops/gcp/pipeline/override-stages.yml +++ b/devops/gcp/pipeline/override-stages.yml @@ -2,7 +2,6 @@ variables: OSDU_GCP_ENABLE_BOOTSTRAP: "true" OSDU_GCP_SERVICE: schema OSDU_GCP_VENDOR: gcp - OSDU_GCP_TESTS_SUBDIR: testing/schema-test-core OSDU_GCP_HELM_CONFIG_SERVICE: schema-config OSDU_GCP_HELM_DEPLOYMENT_SERVICE: schema-deploy OSDU_GCP_HELM_TIMEOUT: "--timeout 15m" @@ -64,18 +63,6 @@ osdu-gcp-containerize-bootstrap-gcr: variables: BUILD_PATH: devops/$OSDU_GCP_VENDOR/bootstrap-osdu-module/Dockerfile -osdu-gcp-test: - script: - - $MAVEN_BUILD . test-results.log verify -q -f $OSDU_GCP_TESTS_SUBDIR/pom.xml - artifacts: - when: always - paths: - - test-results.log - - $OSDU_GCP_TESTS_SUBDIR/target/*/TEST-*.xml - reports: - junit: - - $OSDU_GCP_TESTS_SUBDIR/target/*/TEST-*.xml - osdu-gcp-anthos-deploy-deployment: needs: - osdu-gcp-containerize-gitlab -- GitLab From 30e0b9398d1744aef5c3fba8cb54a2be04f40526 Mon Sep 17 00:00:00 2001 From: "Riabokon Stanislav(EPAM)[GCP]" Date: Sat, 21 May 2022 13:29:06 +0000 Subject: [PATCH 3/6] Update README.md --- provider/schema-gcp/docs/anthos/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/provider/schema-gcp/docs/anthos/README.md b/provider/schema-gcp/docs/anthos/README.md index 0f470328..0c69956c 100644 --- a/provider/schema-gcp/docs/anthos/README.md +++ b/provider/schema-gcp/docs/anthos/README.md @@ -47,7 +47,7 @@ You will need to have the following environment variables defined. | name | value | description | sensitive? | source | | --- | --- | --- | --- | --- | -| `VENDOR` | `gcp` | Use value 'gcp' to run gcp tests | no | - | +| `VENDOR` | `anthos` | Use value 'gcp' to run gcp tests | no | - | | `HOST` | ex`http://localhost:8080` | Schema service host | no | - | | `PRIVATE_TENANT2` | ex`opendes` | OSDU tenant used for testing | no | - | | `PRIVATE_TENANT1` | ex`osdu` | OSDU tenant used for testing | no | - | @@ -482,4 +482,4 @@ For shared tenant only: ListObjects, CRUDObject - \ No newline at end of file + -- GitLab From 85762117dea398c523084e2d247c0044c45851ea Mon Sep 17 00:00:00 2001 From: Mikhail Piatliou Date: Mon, 23 May 2022 12:37:51 +0400 Subject: [PATCH 4/6] GONRG-4968: custom test path for generating xml reports --- devops/gcp/pipeline/override-stages.yml | 37 +++++++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/devops/gcp/pipeline/override-stages.yml b/devops/gcp/pipeline/override-stages.yml index 43324d61..5a7b3c95 100644 --- a/devops/gcp/pipeline/override-stages.yml +++ b/devops/gcp/pipeline/override-stages.yml @@ -55,6 +55,43 @@ osdu-gcp-deploy-deployment: # - echo $STATUS # - if [[ "$STATUS" != *"met"* ]]; then echo "POD didn't start correctly" ; exit 1 ; fi +osdu-gcp-test: + script: + - $MAVEN_BUILD . test-results.log verify -q -f testing/schema-test-core/pom.xml + artifacts: + when: always + paths: + - test-results.log + - testing/schema-test-core/target/*/TEST-*.xml + reports: + junit: + - testing/schema-test-core/target/*/TEST-*.xml + +# FIXME for DEV2 +osdu-gcp-dev2-test: + script: + - $MAVEN_BUILD . test-results.log verify -q -f testing/schema-test-core/pom.xml + artifacts: + when: always + paths: + - test-results.log + - testing/schema-test-core/target/*/TEST-*.xml + reports: + junit: + - testing/schema-test-core/target/*/TEST-*.xml + +osdu-gcp-anthos-test: + script: + - $MAVEN_BUILD . test-results.log verify -q -f testing/schema-test-core/pom.xml + artifacts: + when: always + paths: + - test-results.log + - testing/schema-test-core/target/*/TEST-*.xml + reports: + junit: + - testing/schema-test-core/target/*/TEST-*.xml + osdu-gcp-containerize-bootstrap-gitlab: variables: BUILD_PATH: devops/$OSDU_GCP_VENDOR/bootstrap-osdu-module/Dockerfile -- GitLab From 7e2c425985dbc547e78322f8601d5fce30068667 Mon Sep 17 00:00:00 2001 From: Mikhail Piatliou Date: Mon, 23 May 2022 16:50:11 +0400 Subject: [PATCH 5/6] GONRG-4968: extended helm timeout for schemas bootstrap --- devops/gcp/pipeline/override-stages.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/devops/gcp/pipeline/override-stages.yml b/devops/gcp/pipeline/override-stages.yml index 5a7b3c95..f7fdcb8b 100644 --- a/devops/gcp/pipeline/override-stages.yml +++ b/devops/gcp/pipeline/override-stages.yml @@ -4,7 +4,7 @@ variables: OSDU_GCP_VENDOR: gcp OSDU_GCP_HELM_CONFIG_SERVICE: schema-config OSDU_GCP_HELM_DEPLOYMENT_SERVICE: schema-deploy - OSDU_GCP_HELM_TIMEOUT: "--timeout 15m" + OSDU_GCP_HELM_TIMEOUT: "--timeout 20m" OSDU_GCP_HELM_CONFIG_SERVICE_VARS: > --set data.log_level=INFO --set data.google_audiences=$GOOGLE_AUDIENCE -- GitLab From 7e5dad7563612a076c87ac5a0ee028683f2a3a20 Mon Sep 17 00:00:00 2001 From: Mikhail Piatliou Date: Mon, 23 May 2022 19:17:16 +0400 Subject: [PATCH 6/6] GONRG-4968: finalized changes --- .gitlab-ci.yml | 1 - devops/gcp/pipeline/override-stages.yml | 30 ++++++++++++------------- 2 files changed, 15 insertions(+), 16 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 471ed375..c570e48d 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -62,7 +62,6 @@ include: file: "cloud-providers/azure.yml" - project: "osdu/platform/ci-cd-pipelines" - ref: "gcp-int-tests-anthos" file: "cloud-providers/osdu-gcp-gke.yml" - local: "devops/gcp/pipeline/override-stages.yml" diff --git a/devops/gcp/pipeline/override-stages.yml b/devops/gcp/pipeline/override-stages.yml index f7fdcb8b..fc19392c 100644 --- a/devops/gcp/pipeline/override-stages.yml +++ b/devops/gcp/pipeline/override-stages.yml @@ -4,7 +4,7 @@ variables: OSDU_GCP_VENDOR: gcp OSDU_GCP_HELM_CONFIG_SERVICE: schema-config OSDU_GCP_HELM_DEPLOYMENT_SERVICE: schema-deploy - OSDU_GCP_HELM_TIMEOUT: "--timeout 20m" + OSDU_GCP_HELM_TIMEOUT: "--timeout 15m" OSDU_GCP_HELM_CONFIG_SERVICE_VARS: > --set data.log_level=INFO --set data.google_audiences=$GOOGLE_AUDIENCE @@ -24,6 +24,14 @@ variables: # FIXME add value below for DEV2 pipeline # OSDU_GCP_HELM_DEPLOYMENT_SERVICE_VARS_DEV2: > +osdu-gcp-containerize-bootstrap-gitlab: + variables: + BUILD_PATH: devops/$OSDU_GCP_VENDOR/bootstrap-osdu-module/Dockerfile + +osdu-gcp-containerize-bootstrap-gcr: + variables: + BUILD_PATH: devops/$OSDU_GCP_VENDOR/bootstrap-osdu-module/Dockerfile + # REFACTOR to common pipeline osdu-gcp-deploy-deployment: needs: @@ -55,6 +63,12 @@ osdu-gcp-deploy-deployment: # - echo $STATUS # - if [[ "$STATUS" != *"met"* ]]; then echo "POD didn't start correctly" ; exit 1 ; fi +osdu-gcp-anthos-deploy-deployment: + needs: + - osdu-gcp-containerize-gitlab + - osdu-gcp-containerize-bootstrap-gitlab + - osdu-gcp-anthos-deploy-configmap + osdu-gcp-test: script: - $MAVEN_BUILD . test-results.log verify -q -f testing/schema-test-core/pom.xml @@ -91,17 +105,3 @@ osdu-gcp-anthos-test: reports: junit: - testing/schema-test-core/target/*/TEST-*.xml - -osdu-gcp-containerize-bootstrap-gitlab: - variables: - BUILD_PATH: devops/$OSDU_GCP_VENDOR/bootstrap-osdu-module/Dockerfile - -osdu-gcp-containerize-bootstrap-gcr: - variables: - BUILD_PATH: devops/$OSDU_GCP_VENDOR/bootstrap-osdu-module/Dockerfile - -osdu-gcp-anthos-deploy-deployment: - needs: - - osdu-gcp-containerize-gitlab - - osdu-gcp-containerize-bootstrap-gitlab - - osdu-gcp-anthos-deploy-configmap -- GitLab