Skip to content
Snippets Groups Projects
Commit becde17d authored by Riabokon Stanislav(EPAM)[GCP]'s avatar Riabokon Stanislav(EPAM)[GCP]
Browse files

Merge branch 'master' into 'integration-master' 

Sync updates from master

See merge request go3-nrg/platform/System/Register!9
parents d6b45581 a67c6191
No related branches found
No related tags found
4 merge requests!73Register Service: Audit Logs Implementation (GONRG-1761),!71Logging Enhancements for GCP modules (GONRG-1735, GONRG-1779),!67GCP release/0.5 - fix GSA Challenge (GONRG-1796),!56Gcp fix sonar comments (GONRG-1370)
Hide whitespace changes
Inline Side-by-side
Showing
with 518 additions and 161 deletions
.gitlab-ci.yml
+ 17
− 1
View file @ becde17d
variables:
OSDU_GCP_APPLICATION_NAME: os-register
OSDU_GCP_VENDOR: gcp
ENVIRONMENT: dev
OSDU_GCP_SERVICE: register
OSDU_GCP_ENV_VARS: GOOGLE_CLOUD_PROJECT=${OSDU_GCP_PROJECT},ENTITLEMENTS_API=${OSDU_GCP_ENTITLEMENTS_URL},GCLOUD_REGION=${OSDU_GCP_CLOUDRUN_REGION},STORAGE_API=${OSDU_GCP_STORAGE_URL},INTEGRATION_TEST_AUDIENCES=${GOOGLE_AUDIENCE},SUBSCRIBER_SECRET=${OSDU_GCP_SUBSCRIBER_SECRET},SUBSCRIBER_PRIVATE_KEY_ID=${OSDU_GCP_SUBSCRIBER_PRIVATE_KEY_ID},ENVIRONMENT=${ENVIRONMENT},CRON_JOB_EXPECTED_IP=${CRON_JOB_EXPECTED_IP},RECORDS_CHANGE_PUBSUB_ENDPOINT=${RECORDS_CHANGE_PUBSUB_ENDPOINT},SERVICE_IDENTITY=${SERVICE_IDENTITY}
AWS_BUILD_SUBDIR: provider/register-aws/build-aws
AWS_TEST_SUBDIR: testing/register-test-aws
AWS_SERVICE: register
AWS_ENVIRONMENT: dev
IBM_BUILD_SUBDIR: provider/register-ibm
IBM_INT_TEST_SUBDIR: testing/register-test-ibm
AZURE_SERVICE: register
AZURE_BUILD_SUBDIR: provider/register-azure
AZURE_TEST_SUBDIR: testing/register-test-azure
include:
- project: "osdu/platform/ci-cd-pipelines"
file: "standard-setup.yml"
......@@ -29,6 +39,12 @@ include:
- project: "osdu/platform/ci-cd-pipelines"
file: "scanners/gitlab-ultimate.yml"
- project: 'osdu/platform/ci-cd-pipelines'
file: 'cloud-providers/osdu-gcp-cloudrun.yml'
- project: 'osdu/platform/ci-cd-pipelines'
ref: master
file: 'cloud-providers/azure.yml'
# disable the eslint scanner
# I think this is being generated from the presence of an HTML file, but there
......
devops/azure/README.md 0 → 100644
+ 17
− 0
View file @ becde17d
# Pipeline Support Commands
```bash
AZURE_SERVICE="register"
REPO_BRANCH="master"
TAG="latest"
PARTIAL=${REPO_BRANCH/\//-}
BRANCH=${PARTIAL/./-}
echo "--set image.branch=$BRANCH --set image.tag=$TAG"
# Install the Service
helm upgrade -i osdu-gitlab-$AZURE_SERVICE chart --set image.branch=$BRANCH --set image.tag=$TAG
pod=$(kubectl get pod |grep $AZURE_SERVICE | tail -1 | awk '{print $1}')
status=$(kubectl wait --for=condition=Ready pod/$pod --timeout=60s)
if [[ "$status" != *"met"* ]]; then echo "POD didn't start correctly" ; exit 1 ; fi
```
devops/azure/chart/Chart.yaml 0 → 100644
+ 20
− 0
View file @ becde17d
# Copyright © Microsoft Corporation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
apiVersion: v1
name: register
appVersion: "latest"
description: Helm Chart for installing register service.
version: 0.1.0
type: application
devops/azure/chart/templates/auth.yaml 0 → 100644
+ 32
− 0
View file @ becde17d
# Copyright © Microsoft Corporation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
apiVersion: security.istio.io/v1beta1
kind: AuthorizationPolicy
metadata:
name: register-jwt-auth
namespace: osdu
spec:
selector:
matchLabels:
app: register
action: DENY
rules:
- from:
- source:
notRequestPrincipals: ["*"]
to:
- operation:
notPaths: ["/","*/swagger-resources","*/swagger",
"/api/register/v1/swagger-resources/*","*/swagger-ui.html","*/actuator/health","/api/register/v1/test/challenge/*"]
\ No newline at end of file
devops/azure/chart/templates/deployment.yaml 0 → 100644
+ 98
− 0
View file @ becde17d
# Copyright © Microsoft Corporation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
apiVersion: apps/v1
kind: Deployment
metadata:
name: {{ .Release.Name }}
namespace: osdu
spec:
replicas: {{ .Values.global.replicaCount }}
selector:
matchLabels:
app: {{ .Chart.Name }}
template:
metadata:
labels:
app: {{ .Chart.Name }}
aadpodidbinding: osdu-identity
spec:
volumes:
- name: azure-keyvault
csi:
driver: secrets-store.csi.k8s.io
readOnly: true
volumeAttributes:
secretProviderClass: azure-keyvault
containers:
- name: {{ .Chart.Name }}
image: {{ .Values.image.repository }}/{{ .Chart.Name }}-{{ .Values.image.branch }}:{{ .Values.image.tag | default .Chart.AppVersion }}
imagePullPolicy: Always
ports:
- containerPort: 80
readinessProbe:
httpGet:
path: /api/register/v1/actuator/health
port: 80
volumeMounts:
- name: azure-keyvault
mountPath: "/mnt/azure-keyvault"
readOnly: true
env:
- name: spring_application_name
value: register
- name: LOG_PREFIX
value: "register"
- name: server.servlet.contextPath
value: /api/register/v1/
- name: server.port
value: "80"
- name: KEYVAULT_URI
valueFrom:
configMapKeyRef:
name: osdu-svc-properties
key: ENV_KEYVAULT
- name: AZURE_CLIENT_ID
valueFrom:
secretKeyRef:
name: active-directory
key: principal-clientid
- name: AZURE_CLIENT_SECRET
valueFrom:
secretKeyRef:
name: active-directory
key: principal-clientpassword
- name: AZURE_TENANT_ID
valueFrom:
secretKeyRef:
name: active-directory
key: tenantid
- name: aad_client_id
valueFrom:
secretKeyRef:
name: active-directory
key: application-appid
- name: appinsights_key
valueFrom:
secretKeyRef:
name: central-logging
key: appinsights
- name: cosmosdb_database
value: osdu-db
- name: ENTITLEMENTS_API
value: http://entitlements-azure/entitlements/v1
- name: RECORDS_CHANGE_PUBSUB_ENDPOINT
value: https://haaggarw-eventgrid-viewer.azurewebsites.net/api/updates
- name: SUBSCRIBER_SECRET
value: "395f1b05e95171d7c0dde0b19fd6cf"
devops/azure/chart/templates/service.yaml 0 → 100644
+ 27
− 0
View file @ becde17d
# Copyright © Microsoft Corporation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
apiVersion: v1
kind: Service
metadata:
name: {{ .Chart.Name }}
namespace: osdu
spec:
type: ClusterIP
ports:
- protocol: TCP
port: 80
targetPort: 80
selector:
app: {{ .Chart.Name }}
devops/azure/chart/values.yaml 0 → 100644
+ 21
− 0
View file @ becde17d
# Copyright © Microsoft Corporation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
global:
replicaCount: 1
image:
repository: community.opengroup.org:5555/osdu/platform/system/register
branch: master
tag: latest
pom.xml
+ 20
− 0
View file @ becde17d
......@@ -107,4 +107,24 @@
</snapshotRepository>
</distributionManagement>
<profiles>
<profile>
<id>Default</id>
<activation>
<property>
<name>!repo.releases.id</name>
</property>
</activation>
<properties>
<repo.releases.id>community-maven-repo</repo.releases.id>
<publish.snapshots.id>community-maven-via-job-token</publish.snapshots.id>
<publish.releases.id>community-maven-via-job-token</publish.releases.id>
<repo.releases.url>https://community.opengroup.org/api/v4/groups/17/-/packages/maven</repo.releases.url>
<publish.snapshots.url>https://community.opengroup.org/api/v4/projects/157/packages/maven</publish.snapshots.url>
<publish.releases.url>https://community.opengroup.org/api/v4/projects/157/packages/maven</publish.releases.url>
</properties>
</profile>
</profiles>
</project>
provider/register-aws/pom.xml
+ 1
− 19
View file @ becde17d
......@@ -29,7 +29,7 @@
<groupId>org.opengroup.osdu</groupId>
<artifactId>os-register</artifactId>
<version>1.0.0</version>
<relativePath>../../</relativePath>
<relativePath>../../pom.xml</relativePath>
</parent>
<properties>
<aws.version>1.11.637</aws.version>
......@@ -131,24 +131,6 @@
</dependencies>
<repositories>
<repository>
<id>${gitlab-server}</id>
<url>https://community.opengroup.org/api/v4/groups/17/-/packages/maven</url>
</repository>
</repositories>
<distributionManagement>
<repository>
<id>${gitlab-server}</id>
<url>https://community.opengroup.org/api/v4/projects/157/packages/maven</url>
</repository>
<snapshotRepository>
<id>${gitlab-server}</id>
<url>https://community.opengroup.org/api/v4/projects/157/packages/maven</url>
</snapshotRepository>
</distributionManagement>
<build>
<plugins>
<plugin>
......
provider/register-azure/pom.xml
+ 3
− 3
View file @ becde17d
......@@ -46,9 +46,9 @@
<version>4.1.4</version>
</dependency>
<dependency>
<groupId>com.microsoft.azure.eventgrid.v2019_01_01</groupId>
<groupId>com.microsoft.azure.eventgrid.v2020_04_01_preview</groupId>
<artifactId>azure-mgmt-eventgrid</artifactId>
<version>1.0.0</version>
<version>1.0.0-beta-3</version>
</dependency>
<dependency>
<groupId>com.microsoft.azure</groupId>
......@@ -90,7 +90,7 @@
<dependency>
<groupId>org.opengroup.osdu</groupId>
<artifactId>core-lib-azure</artifactId>
<version>0.0.23</version>
<version>0.0.28</version>
</dependency>
<dependency>
<groupId>org.opengroup.osdu</groupId>
......
provider/register-azure/src/main/java/org/opengroup/osdu/register/provider/azure/di/AzureBootstrapConfig.java
+ 46
− 19
View file @ becde17d
......@@ -14,6 +14,7 @@
package org.opengroup.osdu.register.provider.azure.di;
import com.azure.core.credential.TokenCredential;
import com.azure.identity.DefaultAzureCredentialBuilder;
import com.azure.security.keyvault.keys.cryptography.CryptographyClient;
import com.azure.security.keyvault.keys.cryptography.CryptographyClientBuilder;
......@@ -22,7 +23,7 @@ import com.azure.security.keyvault.secrets.models.KeyVaultSecret;
import com.microsoft.azure.AzureEnvironment;
import com.microsoft.azure.credentials.ApplicationTokenCredentials;
import com.microsoft.azure.credentials.AzureTokenCredentials;
import com.microsoft.azure.management.eventgrid.v2019_01_01.implementation.EventGridManager;
import com.microsoft.azure.management.eventgrid.v2020_04_01_preview.implementation.EventGridManager;
import com.microsoft.rest.LogLevel;
import lombok.Getter;
import org.springframework.beans.factory.annotation.Value;
......@@ -41,9 +42,6 @@ public class AzureBootstrapConfig {
@Value("${azure.cosmosdb.database}")
private String cosmosDBName;
@Value("${azure.cryptoKey.identifier}")
private String keyIdentifier;
@Value("${azure.clientId}")
private String azureClientId;
......@@ -53,12 +51,17 @@ public class AzureBootstrapConfig {
@Value("${azure.tenantId}")
private String azureTenantId;
@Value("${azure.subscriptionId}")
@Value("${azure.appResourceId}")
private String AzureAppResourceId;
private String keyIdentifier;
private String azureSubscriptionId;
@Value("${azure.resourceGroupName}")
private String resourceGroupName;
private String eventGridTopicName;
@Bean
@Named("KEY_VAULT_URL")
public String keyVaultURL() {
......@@ -71,37 +74,60 @@ public class AzureBootstrapConfig {
return cosmosDBName;
}
private void setEventGridTopicName(SecretClient kv) {
eventGridTopicName = getKeyVaultSecret(kv, "opendes-eventgrid-recordstopic").split("\\.")[0].replace("https://", "");
}
private void setResourceGroupName(SecretClient kv) {
resourceGroupName = getKeyVaultSecret(kv, "opendes-eventgrid-resourcegroup");
}
private void setAzureSubscriptionId(SecretClient kv) {
azureSubscriptionId = getKeyVaultSecret(kv, "subscription-id");
}
private void setKeyIdentifier(SecretClient kv) {
keyIdentifier = getKeyVaultSecret(kv, "opendes-encryption-key-identifier");
}
@Bean
@Named("COSMOS_ENDPOINT")
public String cosmosEndpoint(SecretClient kv) {
return getKeyVaultSecret(kv, "cosmos-endpoint");
return getKeyVaultSecret(kv, "opendes-cosmos-endpoint");
}
@Bean
@Named("COSMOS_KEY")
public String cosmosKey(SecretClient kv) {
return getKeyVaultSecret(kv, "cosmos-primary-key");
return getKeyVaultSecret(kv, "opendes-cosmos-primary-key");
}
@Bean
public CryptographyClient getCryptographyClient() {
public CryptographyClient getCryptographyClient(SecretClient kv) {
setKeyIdentifier(kv);
setAzureSubscriptionId(kv);
TokenCredential credential = new DefaultAzureCredentialBuilder().build();
return new CryptographyClientBuilder()
.keyIdentifier(keyIdentifier)
.credential(new DefaultAzureCredentialBuilder().build())
.credential(credential)
.buildClient();
}
@Bean
public EventGridManager eventGridManager() {
AzureTokenCredentials azureTokenCredentials = getAzureTokenCredentials();
return EventGridManager
.configure()
.withLogLevel(LogLevel.BASIC)
.authenticate(azureTokenCredentials, azureTokenCredentials.defaultSubscriptionId());
}
@Bean
public EventGridManager eventGridManager(SecretClient kv) {
setResourceGroupName(kv);
setAzureSubscriptionId(kv);
setEventGridTopicName(kv);
AzureTokenCredentials azureTokenCredentials = getAzureTokenCredentials();
return EventGridManager
.configure()
.withLogLevel(LogLevel.BASIC)
.authenticate(azureTokenCredentials, azureTokenCredentials.defaultSubscriptionId());
}
private AzureTokenCredentials getAzureTokenCredentials() {
private AzureTokenCredentials getAzureTokenCredentials() {
AzureEnvironment azureEnvironment = new AzureEnvironment(new HashMap<>());
azureEnvironment.endpoints().putAll(AzureEnvironment.AZURE.endpoints());
return new ApplicationTokenCredentials(
......@@ -125,4 +151,5 @@ public class AzureBootstrapConfig {
return secretValue;
}
}
\ No newline at end of file
provider/register-azure/src/main/java/org/opengroup/osdu/register/provider/azure/security/AADSecurityConfig.java deleted 100644 → 0
+ 0
− 52
View file @ d6b45581
// Copyright © Microsoft Corporation
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package org.opengroup.osdu.register.provider.azure.security;
import com.microsoft.azure.spring.autoconfigure.aad.AADAppRoleStatelessAuthenticationFilter;
import org.springframework.security.config.annotation.method.configuration.EnableGlobalMethodSecurity;
import org.springframework.security.config.annotation.web.builders.HttpSecurity;
import org.springframework.security.config.annotation.web.configuration.EnableWebSecurity;
import org.springframework.security.config.annotation.web.configuration.WebSecurityConfigurerAdapter;
import org.springframework.security.config.http.SessionCreationPolicy;
import org.springframework.security.web.authentication.UsernamePasswordAuthenticationFilter;
import javax.inject.Inject;
@EnableWebSecurity
@EnableGlobalMethodSecurity(prePostEnabled = true)
public class AADSecurityConfig extends WebSecurityConfigurerAdapter {
@Inject
private AADAppRoleStatelessAuthenticationFilter appRoleAuthFilter;
@Override
protected void configure(HttpSecurity http) throws Exception {
http
.csrf().disable()
.sessionManagement().sessionCreationPolicy(SessionCreationPolicy.NEVER)
.and()
.authorizeRequests()
.antMatchers("/", "/index.html",
"/v2/api-docs",
"/configuration/ui",
"/swagger-resources/**",
"/configuration/security",
"/swagger",
"/swagger-ui.html",
"/webjars/**").permitAll()
.anyRequest().authenticated()
.and()
.addFilterBefore(appRoleAuthFilter, UsernamePasswordAuthenticationFilter.class);
}
}
\ No newline at end of file
provider/register-azure/src/main/java/org/opengroup/osdu/register/provider/azure/security/AzureIstioSecurityConfig.java 0 → 100644
+ 31
− 0
View file @ becde17d
// Copyright © Microsoft Corporation
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package org.opengroup.osdu.register.provider.azure.security;
import org.springframework.security.config.annotation.method.configuration.EnableGlobalMethodSecurity;
import org.springframework.security.config.annotation.web.builders.HttpSecurity;
import org.springframework.security.config.annotation.web.configuration.EnableWebSecurity;
import org.springframework.security.config.annotation.web.configuration.WebSecurityConfigurerAdapter;
@EnableWebSecurity
@EnableGlobalMethodSecurity(prePostEnabled = true)
public class AzureIstioSecurityConfig extends WebSecurityConfigurerAdapter {
@Override
protected void configure(HttpSecurity http) throws Exception {
http.httpBasic().disable()
.csrf().disable(); //AuthN is disabled. AuthN is handled by sidecar proxy
}
}
provider/register-azure/src/main/java/org/opengroup/osdu/register/provider/azure/subscriber/PushSubscription.java
+ 28
− 19
View file @ becde17d
......@@ -2,8 +2,8 @@ package org.opengroup.osdu.register.provider.azure.subscriber;
import com.microsoft.azure.CloudException;
import com.microsoft.azure.arm.model.Indexable;
import com.microsoft.azure.management.eventgrid.v2019_01_01.WebHookEventSubscriptionDestination;
import com.microsoft.azure.management.eventgrid.v2019_01_01.implementation.EventGridManager;
import com.microsoft.azure.management.eventgrid.v2020_04_01_preview.WebHookEventSubscriptionDestination;
import com.microsoft.azure.management.eventgrid.v2020_04_01_preview.implementation.EventGridManager;
import org.opengroup.osdu.core.common.logging.JaxRsDpsLog;
import org.opengroup.osdu.core.common.model.http.AppException;
import org.opengroup.osdu.register.provider.azure.di.AzureBootstrapConfig;
......@@ -11,12 +11,12 @@ import org.opengroup.osdu.register.utils.AppServiceConfig;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicReference;
@Component
public class PushSubscription {
private static final String RESOURCE_PROVISIONING_ERROR_MESSAGE = "Resource cannot be updated during provisioning";
@Autowired
private EventGridManager eventGridManager;
......@@ -29,12 +29,14 @@ public class PushSubscription {
@Autowired
private AppServiceConfig serviceConfig;
public boolean createPushSubscription(String subscriptionId, String topicName) {
public void createPushSubscription(String subscriptionId, String topicName) {
WebHookEventSubscriptionDestination subscriptionDestination = new WebHookEventSubscriptionDestination();
subscriptionDestination.withEndpointUrl(serviceConfig.getRecordsChangePubsubEndpoint());
subscriptionDestination.withAzureActiveDirectoryTenantId(azureBootstrapConfig.getAzureTenantId());
subscriptionDestination.withAzureActiveDirectoryApplicationIdOrUri(azureBootstrapConfig.getAzureAppResourceId());
String scope = String.format("/subscriptions/%s/resourceGroups/%s/providers/Microsoft.EventGrid/topics/%s", azureBootstrapConfig.getAzureSubscriptionId(), azureBootstrapConfig.getResourceGroupName(), topicName);
AtomicBoolean res = new AtomicBoolean(true);
AtomicReference<Throwable> error = new AtomicReference<>();
eventGridManager.eventSubscriptions().define(subscriptionId)
.withScope(scope)
......@@ -43,21 +45,28 @@ public class PushSubscription {
.toBlocking()
.subscribe(
(Indexable indexable) -> {},
(Throwable throwable) -> {
logger.error(String.format("Creating subscription with id %s failed with error %s", subscriptionId, throwable.getMessage()));
res.set(false);
},
() -> logger.info(String.format("Subscription with id %s created successfully", subscriptionId))
error::set,
() -> logger.info(String.format("Push Subscription with id %s created successfully", subscriptionId))
);
return res.get();
if(error.get() != null) {
if(error.get().getMessage().equals(RESOURCE_PROVISIONING_ERROR_MESSAGE)) {
logger.error("Another request is trying to create the same Push subscription");
throw new AppException(409, "Conflict", "Another request is trying to create the same Push subscription");
}
else {
logger.error("Creating Push Subscription failed with error: " + error.get().toString());
throw new AppException(500, "Server Error", "Unexpected error creating Push subscription");
}
}
}
public void deletePushSubscription(String subscriptionId, String topicName) {
String scope = String.format("/subscriptions/%s/resourceGroups/%s/providers/Microsoft.EventGrid/topics/%s", azureBootstrapConfig.getAzureSubscriptionId(), azureBootstrapConfig.getResourceGroupName(), topicName);
AtomicReference<Throwable> error = new AtomicReference<>();
eventGridManager.eventSubscriptions().deleteAsync(scope, subscriptionId)
.subscribe(() -> logger.info(String.format("Subscription with id %s deleted successfully", subscriptionId)), error::set);
.subscribe(() -> logger.info(String.format("Push Subscription with id %s deleted successfully", subscriptionId)), error::set);
if(error.get() != null) {
if(error.get() instanceof CloudException) {
......@@ -66,8 +75,8 @@ public class PushSubscription {
throw new AppException(cloudException.response().code(), cloudException.body().code(), cloudException.body().message());
}
else {
logger.error(error.get().toString());
throw new AppException(500, "Server Error", "Unexpected error deleting subscription");
logger.error("Deleting Push Subscription failed with error: " + error.get().toString());
throw new AppException(500, "Server Error", "Unexpected error deleting Push subscription");
}
}
}
......@@ -79,17 +88,17 @@ public class PushSubscription {
.subscribe(
(Indexable indexable) -> {},
error::set,
() -> logger.info(String.format("Subscription with id %s fetched successfully", subscriptionId))
() -> logger.info(String.format("Push Subscription with id %s fetched successfully", subscriptionId))
);
if(error.get() != null) {
if(error.get() instanceof NullPointerException) {
logger.error(String.format("Subscriber with id %s does not exist.", subscriptionId));
throw new AppException(404, "Not found", String.format("Subscriber with id %s does not exist.", subscriptionId));
logger.error(String.format("Push Subscription with id %s does not exist.", subscriptionId));
throw new AppException(404, "Not found", String.format("Push Subscription with id %s does not exist.", subscriptionId));
}
else {
logger.error(error.get().toString());
throw new AppException(500, "Server Error", "Unexpected error deleting subscription");
logger.error("Fetching Push subscription failed with error: " + error.get().toString());
throw new AppException(500, "Server Error", "Unexpected error while Fetching Push subscription");
}
}
}
......
provider/register-azure/src/main/java/org/opengroup/osdu/register/provider/azure/subscriber/SubscriptionRepository.java
+ 28
− 15
View file @ becde17d
......@@ -90,6 +90,7 @@ public class SubscriptionRepository implements ISubscriptionRepository {
* be different like description etc.
*
*/
@Override
public Subscription create(Subscription input) {
......@@ -101,6 +102,7 @@ public class SubscriptionRepository implements ISubscriptionRepository {
try {
cosmosStore.createItem(dpsHeaders.getPartitionId(), azureBootstrapConfig.getCosmosDBName(), cosmosContainerConfig.getSubscriptionContainerName(), doc);
logger.info(String.format("Record successfully created for Subscription with id %s", input.getId()));
}
catch (AppException e) {
if(e.getError().getCode() == 409) {
......@@ -108,6 +110,7 @@ public class SubscriptionRepository implements ISubscriptionRepository {
// and deleting the corresponding record from Cosmos Db was also unsuccessful.
// This will result in an 500 Exception so the user should be able to create the subscription with
// the same topic and pushEndpoint combination again
logger.info(String.format("Record already exists for Subscription with id %s", input.getId()));
SubscriptionDoc output = createPushSubscriptionIfDoesNotExist(doc);
input.setNotificationId(output.getNotificationId());
return input;
......@@ -118,16 +121,20 @@ public class SubscriptionRepository implements ISubscriptionRepository {
}
}
boolean isSubscriptionCreated = pushSubscription.createPushSubscription(doc.getNotificationId(), doc.getTopic());
if(isSubscriptionCreated) {
try {
pushSubscription.createPushSubscription(doc.getNotificationId(), doc.getTopic());
logger.info("Push Subscription created with Event Grid ID:" + doc.getNotificationId());
return input;
}
else {
catch(AppException e) {
if(e.getError().getCode() == 409) {
throw new AppException(409, "Conflict", "Another request is trying to create the same subscription");
}
try {
cosmosStore.deleteItem(dpsHeaders.getPartitionId(), azureBootstrapConfig.getCosmosDBName(), cosmosContainerConfig.getSubscriptionContainerName(), input.getId(), dpsHeaders.getPartitionId());
logger.info("Record deleted for subscription with ID: " + input.getId());
}
catch (AppException e) {}
catch (AppException ignored) {}
logger.error("Unexpected error creating subscription");
throw new AppException(500, "Server Error", "Unexpected error creating subscription");
......@@ -174,14 +181,16 @@ public class SubscriptionRepository implements ISubscriptionRepository {
Optional<SubscriptionDoc> subscription = cosmosStore.findItem(dpsHeaders.getPartitionId(), azureBootstrapConfig.getCosmosDBName(), cosmosContainerConfig.getSubscriptionContainerName(), id, dpsHeaders.getPartitionId(), SubscriptionDoc.class);
if(!subscription.isPresent()){
logger.info("Record not found for subscription with ID: " + id);
return false;
}
logger.info("Record found for subscription with ID: " + id);
try {
pushSubscription.deletePushSubscription(subscription.get().getNotificationId(), subscription.get().getTopic());
logger.info("Push Subscription deleted with Event Grid ID:" + subscription.get().getNotificationId());
try {
cosmosStore.deleteItem(dpsHeaders.getPartitionId(), azureBootstrapConfig.getCosmosDBName(), cosmosContainerConfig.getSubscriptionContainerName(), id, dpsHeaders.getPartitionId());
logger.info("Record deleted for subscription with ID: " + id);
return true;
}
catch (AppException e) {
......@@ -233,6 +242,7 @@ public class SubscriptionRepository implements ISubscriptionRepository {
// notification-id to create the Push Subscription
Optional<SubscriptionDoc> originalDoc = cosmosStore.findItem(dpsHeaders.getPartitionId(), azureBootstrapConfig.getCosmosDBName(), cosmosContainerConfig.getSubscriptionContainerName(), input.getId(), dpsHeaders.getPartitionId(), SubscriptionDoc.class);
originalDoc.ifPresent(subscriptionDoc -> input.setNotificationId(subscriptionDoc.getNotificationId()));
logger.info(String.format("Creating Push Subscription with id %s if does not exists", input.getNotificationId()));
// We will check if Push Subscription does not exist then we should try creating it again since
// the corresponding record in the Cosmos Db is already present
......@@ -249,19 +259,22 @@ public class SubscriptionRepository implements ISubscriptionRepository {
// If Push Subscription is not yet created we should create it, also we might
// need to update the cosmos db record since some other fields might be different
cosmosStore.upsertItem(dpsHeaders.getPartitionId(), azureBootstrapConfig.getCosmosDBName(), cosmosContainerConfig.getSubscriptionContainerName(), input);
logger.info("Updating Record for subscription with ID: ", input.getId());
// Now creating the Push Subscription, if it fails again we should throw server error
// otherwise we can return Success code as response
boolean isSubscriptionCreated = pushSubscription.createPushSubscription(input.getNotificationId(), input.getTopic());
if(!isSubscriptionCreated) {
throw new AppException(500, "Server Error", "Unexpected error creating subscription");
}
pushSubscription.createPushSubscription(input.getNotificationId(), input.getTopic());
logger.info("Push Subscription created with Event Grid ID:" + input.getNotificationId());
return input;
}
catch (AppException exception) {
logger.error(exception.getError().getMessage());
throw new AppException(500, "Server Error", "Unexpected error creating subscription");
if(exception.getError().getCode() == 409) {
logger.error("Another request is trying to create the same Push subscription");
throw new AppException(409, "Conflict", "Another request is trying to create the same Push subscription");
}
else {
logger.error("Unexpected error creating subscription");
throw new AppException(500, "Server Error", "Unexpected error creating subscription");
}
}
}
else {
......
provider/register-azure/src/main/java/org/opengroup/osdu/register/provider/azure/subscriber/TopicsRepositoryImpl.java 0 → 100644
+ 63
− 0
View file @ becde17d
package org.opengroup.osdu.register.provider.azure.subscriber;
import com.google.gson.Gson;
import com.google.gson.reflect.TypeToken;
import org.apache.http.HttpStatus;
import org.opengroup.osdu.core.common.logging.JaxRsDpsLog;
import org.opengroup.osdu.core.common.model.http.AppException;
import org.opengroup.osdu.core.common.model.notification.Topic;
import org.opengroup.osdu.register.provider.azure.di.AzureBootstrapConfig;
import org.opengroup.osdu.register.provider.interfaces.subscriber.ITopicsRepository;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.annotation.Primary;
import org.springframework.core.io.ClassPathResource;
import org.springframework.stereotype.Repository;
import java.io.ByteArrayOutputStream;
import java.io.InputStream;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.List;
@Repository
@Primary
public class TopicsRepositoryImpl implements ITopicsRepository {
@Autowired
private JaxRsDpsLog log;
@Autowired
private AzureBootstrapConfig config;
private List<Topic> topics;
public List<Topic> listMessages() {
if (topics == null) {
String topicName = config.getEventGridTopicName();
Gson gson = new Gson();
java.lang.reflect.Type listType = new TypeToken<ArrayList<Topic>>() {
}.getType();
topics = gson.fromJson(getFile(), listType);
Topic topic = topics.get(0);
topic.setName(topicName);
topics.set(0, topic);
}
return topics;
}
private String getFile() {
final String fileName = "topics.json";
try {
InputStream inputStream = new ClassPathResource(fileName).getInputStream();
ByteArrayOutputStream result = new ByteArrayOutputStream();
byte[] buffer = new byte[1024];
int length;
while ((length = inputStream.read(buffer)) != -1) {
result.write(buffer, 0, length);
}
return result.toString(StandardCharsets.UTF_8.name());
} catch (Exception e) {
log.error("Error retrieving topics.json", e);
throw new AppException(HttpStatus.SC_SERVICE_UNAVAILABLE, "Server error", "An unexpected error occurred");
}
}
}
provider/register-azure/src/main/resources/application.properties
+ 7
− 8
View file @ becde17d
......@@ -19,22 +19,16 @@ server.servlet.contextPath=/api/register/v1
service.domain.name=${service_domain_name}
# Azure AD configuration
azure.activedirectory.client-id=${aad_client_id}
azure.activedirectory.AppIdUri=api://${azure.activedirectory.client-id}
azure.activedirectory.session-stateless=true
azure.clientId=${AZURE_CLIENT_ID}
azure.clientSecret=${AZURE_CLIENT_SECRET}
azure.tenantId=${AZURE_TENANT_ID}
azure.subscriptionId=${AZURE_SUBSCRIPTION_ID}
azure.resourceGroupName=${AZURE_RESOURCE_GROUP_NAME}
azure.appResourceId=${aad_client_id}
# Azure CosmosDB configuration
azure.cosmosdb.database=${cosmosdb_database}
# Azure KeyVault configuration
azure.keyvault.url=${KEYVAULT_URI}
azure.cryptoKey.identifier=${KEY_IDENTIFIER}
# Azure App Insights configuration
azure.application-insights.instrumentation-key=${appinsights_key}
......@@ -46,10 +40,15 @@ spring.application.name=register-azure
registerAction.container.name=RegisterAction
registerDdms.container.name=RegisterDdms
registerSubscription.container.name=RegisterSubscription
tenantInfo.container.name=TenantInfo
#logging configuration
logging.transaction.enabled=true
logging.slf4jlogger.enabled=true
logging.mdccontext.enabled=true
INTEGRATION_TEST_AUDIENCES=
CRON_JOB_EXPECTED_IP=0:0:0:0:0:0:0:1
SUBSCRIBER_PRIVATE_KEY_ID=
ENVIRONMENT=LOCAL
management.health.defaults.enabled=false
\ No newline at end of file
provider/register-azure/src/main/resources/topics.json 0 → 100644
+ 30
− 0
View file @ becde17d
[
{
"name": "topic-name",
"description": "This notification is sent whenever a new record or record version is created, updated or deleted, and when a new schema is created in storage.",
"state": "ACTIVE",
"example": [
{
"id": "common:abc:123",
"kind": "common:petrel:regularheightfieldsurface:1.0.0",
"op" : "create",
"recordUpdated": "false"
},
{
"id": "common:abc:124",
"kind": "common:petrel:regularheightfieldsurface:1.0.0",
"op" : "create",
"recordUpdated": "true"
},
{
"kind": "common:petrel:regularheightfieldsurface:1.0.0",
"op" : "create_schema"
},
{
"id": "common:ghi:345",
"kind": "common:petrel:regularheightfieldsurface:1.0.0",
"op" : "delete"
}
]
}
]
\ No newline at end of file
provider/register-azure/src/test/java/org/opengroup/osdu/register/provider/azure/di/AzureBootstrapConfigTest.java
+ 6
− 6
View file @ becde17d
......@@ -50,19 +50,19 @@ public class AzureBootstrapConfigTest {
@Test
public void configReturnsCorrectSecretCosmosKey() {
doReturn("cosmos-key-secret").when(secret).getValue();
doReturn(secret).when(kv).getSecret("cosmos-primary-key");
doReturn("opendes-cosmos-key-secret").when(secret).getValue();
doReturn(secret).when(kv).getSecret("opendes-cosmos-primary-key");
String secretValue = bootstrapConfig.cosmosKey(kv);
assertEquals( "cosmos-key-secret", secretValue);
assertEquals( "opendes-cosmos-key-secret", secretValue);
}
@Test
public void configReturnsCorrectSecretCosmosEndpoint() {
doReturn("cosmos-endpoint-secret").when(secret).getValue();
doReturn(secret).when(kv).getSecret("cosmos-endpoint");
doReturn("opendes-cosmos-endpoint-secret").when(secret).getValue();
doReturn(secret).when(kv).getSecret("opendes-cosmos-endpoint");
String secretValue = bootstrapConfig.cosmosEndpoint(kv);
assertEquals( "cosmos-endpoint-secret", secretValue);
assertEquals( "opendes-cosmos-endpoint-secret", secretValue);
}
}
provider/register-azure/src/test/java/org/opengroup/osdu/register/provider/azure/subscriber/PushSubscriptionTest.java
+ 23
− 19
View file @ becde17d
......@@ -17,9 +17,9 @@ package org.opengroup.osdu.register.provider.azure.subscriber;
import com.microsoft.azure.CloudError;
import com.microsoft.azure.CloudException;
import com.microsoft.azure.arm.model.Indexable;
import com.microsoft.azure.management.eventgrid.v2019_01_01.EventSubscription;
import com.microsoft.azure.management.eventgrid.v2019_01_01.EventSubscriptions;
import com.microsoft.azure.management.eventgrid.v2019_01_01.implementation.EventGridManager;
import com.microsoft.azure.management.eventgrid.v2020_04_01_preview.EventSubscription;
import com.microsoft.azure.management.eventgrid.v2020_04_01_preview.EventSubscriptions;
import com.microsoft.azure.management.eventgrid.v2020_04_01_preview.implementation.EventGridManager;
import okhttp3.ResponseBody;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
......@@ -36,10 +36,8 @@ import rx.Completable;
import rx.Observable;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.Mockito.lenient;
import static org.mockito.Mockito.times;
......@@ -54,6 +52,8 @@ public class PushSubscriptionTest {
private static final String resourceGroupName = "resource-group";
private static final String topicName = "topic-name";
private static final String errorMessage = "some error message";
private static final String tenantId = "tenant-id";
private static final String appResourceId = "app-resource-id";
private static final String scope = String.format("/subscriptions/%s/resourceGroups/%s/providers/Microsoft.EventGrid/topics/%s", subscriptionId, resourceGroupName, topicName);
private final Observable<Indexable> observable = Observable.from(new Indexable[]{});
private final Observable<Indexable> observableError = Observable.error(new Throwable(errorMessage));
......@@ -96,6 +96,8 @@ public class PushSubscriptionTest {
public void init() {
lenient().when(azureBootstrapConfig.getResourceGroupName()).thenReturn(resourceGroupName);
lenient().when(azureBootstrapConfig.getAzureSubscriptionId()).thenReturn(subscriptionId);
lenient().when(azureBootstrapConfig.getAzureTenantId()).thenReturn(tenantId);
lenient().when(azureBootstrapConfig.getAzureAppResourceId()).thenReturn(appResourceId);
lenient().when(serviceConfig.getRecordsChangePubsubEndpoint()).thenReturn(pushEndpoint);
lenient().when(eventGridManager.eventSubscriptions()).thenReturn(eventSubscriptions);
lenient().when(eventSubscriptions.define(subscriptionId)).thenReturn(eventSubscription);
......@@ -104,35 +106,37 @@ public class PushSubscriptionTest {
}
@Test
public void shouldReturnTrueWhenPushSubscriptionCreated() {
public void shouldCreatePushSubscription() {
when(eventSubscriptionWithCreate.createAsync()).thenReturn(observable);
boolean result = pushSubscription.createPushSubscription(subscriptionId, topicName);
pushSubscription.createPushSubscription(subscriptionId, topicName);
assertTrue(result);
verify(azureBootstrapConfig, times(1)).getResourceGroupName();
verify(azureBootstrapConfig, times(1)).getAzureSubscriptionId();
verify(serviceConfig, times(1)).getRecordsChangePubsubEndpoint();
verify(eventGridManager, times(1)).eventSubscriptions();
verify(eventSubscriptions, times(1)).define(subscriptionId);
verify(eventSubscription, times(1)).withScope(scope);
verify(logger, times(1)).info(String.format("Subscription with id %s created successfully", subscriptionId));
verify(logger, times(1)).info(String.format("Push Subscription with id %s created successfully", subscriptionId));
}
@Test
public void shouldReturnFalseWhenPushSubscriptionCreationFailed() {
public void shouldReturn500WhenPushSubscriptionCreationFailed() {
when(eventSubscriptionWithCreate.createAsync()).thenReturn(observableError);
boolean result = pushSubscription.createPushSubscription(subscriptionId, topicName);
assertFalse(result);
AppException exception = assertThrows(AppException.class, () -> {
pushSubscription.createPushSubscription(subscriptionId, topicName);
});
assertNotNull(exception);
assertEquals(500, exception.getError().getCode());
verify(azureBootstrapConfig, times(1)).getResourceGroupName();
verify(azureBootstrapConfig, times(1)).getAzureSubscriptionId();
verify(serviceConfig, times(1)).getRecordsChangePubsubEndpoint();
verify(eventGridManager, times(1)).eventSubscriptions();
verify(eventSubscriptions, times(1)).define(subscriptionId);
verify(eventSubscription, times(1)).withScope(scope);
verify(logger, times(1)).error(String.format("Creating subscription with id %s failed with error %s", subscriptionId, errorMessage));
}
@Test
......@@ -145,7 +149,7 @@ public class PushSubscriptionTest {
verify(azureBootstrapConfig, times(1)).getResourceGroupName();
verify(azureBootstrapConfig, times(1)).getAzureSubscriptionId();
verify(eventGridManager, times(1)).eventSubscriptions();
verify(logger, times(1)).info(String.format("Subscription with id %s deleted successfully", subscriptionId));
verify(logger, times(1)).info(String.format("Push Subscription with id %s deleted successfully", subscriptionId));
}
@Test
......@@ -185,7 +189,7 @@ public class PushSubscriptionTest {
assertEquals(500, exception.getError().getCode());
assertEquals(500, exception.getError().getCode());
assertEquals("Server Error", exception.getError().getReason());
assertEquals("Unexpected error deleting subscription", exception.getError().getMessage());
assertEquals("Unexpected error deleting Push subscription", exception.getError().getMessage());
verify(azureBootstrapConfig, times(1)).getResourceGroupName();
verify(azureBootstrapConfig, times(1)).getAzureSubscriptionId();
verify(eventGridManager, times(1)).eventSubscriptions();
......@@ -201,7 +205,7 @@ public class PushSubscriptionTest {
verify(azureBootstrapConfig, times(1)).getResourceGroupName();
verify(azureBootstrapConfig, times(1)).getAzureSubscriptionId();
verify(eventGridManager, times(1)).eventSubscriptions();
verify(logger, times(1)).info(String.format("Subscription with id %s fetched successfully", subscriptionId));
verify(logger, times(1)).info(String.format("Push Subscription with id %s fetched successfully", subscriptionId));
}
@Test
......@@ -216,11 +220,11 @@ public class PushSubscriptionTest {
assertNotNull(exception);
assertEquals(404, exception.getError().getCode());
assertEquals("Not found", exception.getError().getReason());
assertEquals(String.format("Subscriber with id %s does not exist.", subscriptionId), exception.getError().getMessage());
assertEquals(String.format("Push Subscription with id %s does not exist.", subscriptionId), exception.getError().getMessage());
verify(azureBootstrapConfig, times(1)).getResourceGroupName();
verify(azureBootstrapConfig, times(1)).getAzureSubscriptionId();
verify(eventGridManager, times(1)).eventSubscriptions();
verify(logger, times(1)).error(String.format("Subscriber with id %s does not exist.", subscriptionId));
verify(logger, times(1)).error(String.format("Push Subscription with id %s does not exist.", subscriptionId));
}
@Test
......@@ -235,7 +239,7 @@ public class PushSubscriptionTest {
assertNotNull(exception);
assertEquals(500, exception.getError().getCode());
assertEquals("Server Error", exception.getError().getReason());
assertEquals("Unexpected error deleting subscription", exception.getError().getMessage());
assertEquals("Unexpected error while Fetching Push subscription", exception.getError().getMessage());
verify(azureBootstrapConfig, times(1)).getResourceGroupName();
verify(azureBootstrapConfig, times(1)).getAzureSubscriptionId();
verify(eventGridManager, times(1)).eventSubscriptions();
......
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment