Skip to content
GitLab
Commit bbfbad5b authored by Auto GO3-NRG Ticket's avatar Auto GO3-NRG Ticket
Browse files

Auto commit when pipeline succeeds

parents e7436b1d 58aff352
Hide whitespace changes
Inline Side-by-side
NOTICE
View file @ bbfbad5b
......@@ -23,6 +23,7 @@ The following software have components provided under the terms of this license:
- Apache Commons Logging (from http://commons.apache.org/proper/commons-logging/)
- Apache HttpAsyncClient (from http://hc.apache.org/httpcomponents-asyncclient)
- Apache HttpClient (from http://hc.apache.org/httpcomponents-client)
- Apache HttpClient Cache (from http://hc.apache.org/httpcomponents-client)
- Apache HttpCore (from http://hc.apache.org/httpcomponents-core-ga)
- Apache HttpCore NIO (from http://hc.apache.org/httpcomponents-core-ga)
- Apache Log4j API (from )
......
devops/azure/chart/templates/deployment.yaml
View file @ bbfbad5b
......@@ -63,21 +63,6 @@ spec:
configMapKeyRef:
name: osdu-svc-properties
key: ENV_KEYVAULT
- name: AZURE_CLIENT_ID
valueFrom:
secretKeyRef:
name: active-directory
key: principal-clientid
- name: AZURE_CLIENT_SECRET
valueFrom:
secretKeyRef:
name: active-directory
key: principal-clientpassword
- name: AZURE_TENANT_ID
valueFrom:
secretKeyRef:
name: active-directory
key: tenantid
- name: aad_client_id
valueFrom:
secretKeyRef:
......@@ -90,9 +75,13 @@ spec:
key: appinsights
- name: cosmosdb_database
value: osdu-db
- name: ACCEPT_HTTP # TEMPORARY UNTIL HTTPS
value: "true"
- name: azure_activedirectory_AppIdUri
value: "api://$(aad_client_id)"
- name: ENTITLEMENTS_API
value: http://entitlements-azure/entitlements/v1
- name: RECORDS_CHANGE_PUBSUB_ENDPOINT
value: https://haaggarw-eventgrid-viewer.azurewebsites.net/api/updates
- name: ACCEPT_HTTP # TEMPORARY UNTIL HTTPS
value: "true"
\ No newline at end of file
- name: partition_service_endpoint
value: http://partition/api/partition/v1
\ No newline at end of file
pom.xml
View file @ bbfbad5b
......@@ -32,6 +32,7 @@
<maven.compiler.source>1.8</maven.compiler.source>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<springfox-version>2.7.0</springfox-version>
<osdu.oscorecommon.version>0.3.23</osdu.oscorecommon.version>
</properties>
<licenses>
......@@ -54,7 +55,7 @@
<dependency>
<groupId>org.opengroup.osdu</groupId>
<artifactId>os-core-common</artifactId>
<version>0.3.6</version>
<version>${osdu.oscorecommon.version}</version>
</dependency>
</dependencies>
</dependencyManagement>
......
provider/register-azure/README.md
View file @ bbfbad5b
......@@ -55,6 +55,8 @@ In order to run the service locally, you will need to have the following environ
| `server_port` | ex `8082` | Port the service will run on | no | -- |
| `RECORDS_CHANGE_PUBSUB_ENDPOINT` | ex `https://notification.azurewebsites.net/api/notification/v1/push-handler/records-changed` | This will be the notification service endpoint where record changed events will be pushed | no | -- |
| `ACCEPT_HTTP` | ex `true/false` | to Enable/disable http requests | no | -- |
| `partition_service_endpoint` | ex `https://foo-partition.azurewebsites.net` | Partition Service API endpoint | no | output of infrastructure deployment |
| `azure.activedirectory.app-resource-id` | `********` | AAD client application ID | yes | output of infrastructure deployment |
**Required to run integration tests**
......
provider/register-azure/pom.xml
View file @ bbfbad5b
......@@ -29,21 +29,15 @@
<packaging>jar</packaging>
<version>1.0.0-SNAPSHOT</version>
<properties>
<azure.appservice.resourcegroup></azure.appservice.resourcegroup>
<azure.appservice.plan></azure.appservice.plan>
<azure.appservice.appname></azure.appservice.appname>
<azure.appservice.subscription></azure.appservice.subscription>
<azure.keyvault.keys.version>4.2.3</azure.keyvault.keys.version>
<azure.evengrid.version>1.0.0-beta-3</azure.evengrid.version>
<osdu.corelibazure.version>0.0.48</osdu.corelibazure.version>
<osdu.oscorecommon.version>0.3.16</osdu.oscorecommon.version>
<osdu.register-core.version>1.0.0</osdu.register-core.version>
<osdu.corelibazure.version>0.0.60</osdu.corelibazure.version>
<junit-jupiter.version>5.6.0</junit-jupiter.version>
<mockito-junit-jupiter.version>2.23.0</mockito-junit-jupiter.version>
</properties>
<dependencyManagement>
<dependencies>
<!-- Inherit managed dependencies from core-lib-azure. -->
<!-- Inherit managed dependencies from core-lib-azure -->
<dependency>
<groupId>org.opengroup.osdu</groupId>
<artifactId>core-lib-azure</artifactId>
......@@ -55,20 +49,29 @@
</dependencyManagement>
<dependencies>
<!-- OSDU Dependencies -->
<dependency>
<groupId>com.microsoft.azure</groupId>
<artifactId>azure-active-directory-spring-boot-starter</artifactId>
<groupId>org.opengroup.osdu</groupId>
<artifactId>core-lib-azure</artifactId>
<version>${osdu.corelibazure.version}</version>
</dependency>
<dependency>
<groupId>org.opengroup.osdu</groupId>
<artifactId>os-core-common</artifactId>
</dependency>
<dependency>
<groupId>com.azure</groupId>
<artifactId>azure-security-keyvault-keys</artifactId>
<version>${azure.keyvault.keys.version}</version>
<groupId>org.opengroup.osdu</groupId>
<artifactId>register-core</artifactId>
<version>${osdu.register-core.version}</version>
</dependency>
<!-- Azure Dependencies -->
<dependency>
<groupId>com.microsoft.azure.eventgrid.v2020_04_01_preview</groupId>
<artifactId>azure-mgmt-eventgrid</artifactId>
<version>${azure.evengrid.version}</version>
<groupId>com.microsoft.azure</groupId>
<artifactId>azure-active-directory-spring-boot-starter</artifactId>
</dependency>
<!-- Spring Dependencies -->
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-security</artifactId>
......@@ -101,25 +104,12 @@
<groupId>org.springframework.security</groupId>
<artifactId>spring-security-oauth2-jose</artifactId>
</dependency>
<dependency>
<groupId>org.opengroup.osdu</groupId>
<artifactId>core-lib-azure</artifactId>
<version>${osdu.corelibazure.version}</version>
</dependency>
<dependency>
<groupId>org.opengroup.osdu</groupId>
<artifactId>os-core-common</artifactId>
<version>${osdu.oscorecommon.version}</version>
</dependency>
<dependency>
<groupId>org.opengroup.osdu</groupId>
<artifactId>register-core</artifactId>
<version>${osdu.register-core.version}</version>
</dependency>
<dependency>
<groupId>org.projectlombok</groupId>
<artifactId>lombok</artifactId>
</dependency>
<!-- Test Dependencies -->
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-test</artifactId>
......@@ -134,6 +124,7 @@
<dependency>
<groupId>org.mockito</groupId>
<artifactId>mockito-junit-jupiter</artifactId>
<version>${mockito-junit-jupiter.version}</version>
<scope>test</scope>
</dependency>
</dependencies>
......
provider/register-azure/src/main/java/org/opengroup/osdu/register/provider/azure/di/AzureBootstrapConfig.java
View file @ bbfbad5b
......@@ -14,159 +14,44 @@
package org.opengroup.osdu.register.provider.azure.di;
import com.azure.core.credential.TokenCredential;
import com.azure.cosmos.CosmosClient;
import com.azure.cosmos.CosmosClientBuilder;
import com.azure.identity.DefaultAzureCredentialBuilder;
import com.azure.security.keyvault.keys.cryptography.CryptographyClient;
import com.azure.security.keyvault.keys.cryptography.CryptographyClientBuilder;
import com.azure.security.keyvault.secrets.SecretClient;
import com.azure.security.keyvault.secrets.models.KeyVaultSecret;
import com.microsoft.azure.AzureEnvironment;
import com.microsoft.azure.credentials.ApplicationTokenCredentials;
import com.microsoft.azure.credentials.AzureTokenCredentials;
import com.microsoft.azure.management.eventgrid.v2020_04_01_preview.implementation.EventGridManager;
import com.microsoft.rest.LogLevel;
import lombok.Getter;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Qualifier;
import javax.inject.Inject;
import javax.inject.Named;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import javax.inject.Named;
import java.util.HashMap;
import lombok.Getter;
@Configuration
@Getter
public class AzureBootstrapConfig {
@Value("${azure.keyvault.url}")
private String keyVaultURL;
@Value("${azure.cosmosdb.database}")
private String cosmosDBName;
@Value("${azure.clientId}")
private String azureClientId;
@Value("${azure.clientSecret}")
private String azureClientSecret;
@Value("${azure.tenantId}")
private String azureTenantId;
@Value("${azure.appResourceId}")
private String AzureAppResourceId;
private String keyIdentifier;
private String azureSubscriptionId;
private String resourceGroupName;
private String eventGridTopicName;
@Bean
@Named("KEY_VAULT_URL")
public String keyVaultURL() {
return keyVaultURL;
}
@Bean
@Named("COSMOS_DB_NAME")
public String cosmosDBName() {
return cosmosDBName;
}
private void setEventGridTopicName(SecretClient kv) {
eventGridTopicName = getKeyVaultSecret(kv, "opendes-eventgrid-recordstopic").split("\\.")[0].replace("https://", "");
}
private void setResourceGroupName(SecretClient kv) {
resourceGroupName = getKeyVaultSecret(kv, "opendes-eventgrid-resourcegroup");
}
private void setAzureSubscriptionId(SecretClient kv) {
azureSubscriptionId = getKeyVaultSecret(kv, "subscription-id");
}
private void setKeyIdentifier(SecretClient kv) {
keyIdentifier = getKeyVaultSecret(kv, "opendes-encryption-key-identifier");
}
@Bean
@Named("COSMOS_ENDPOINT")
public String cosmosEndpoint(SecretClient kv) {
return getKeyVaultSecret(kv, "opendes-cosmos-endpoint");
}
@Bean
@Named("COSMOS_KEY")
public String cosmosKey(SecretClient kv) {
return getKeyVaultSecret(kv, "opendes-cosmos-primary-key");
}
@Bean
public CryptographyClient getCryptographyClient(SecretClient kv) {
setKeyIdentifier(kv);
setAzureSubscriptionId(kv);
TokenCredential credential = new DefaultAzureCredentialBuilder().build();
return new CryptographyClientBuilder()
.keyIdentifier(keyIdentifier)
.credential(credential)
.buildClient();
}
@Bean
public EventGridManager eventGridManager(SecretClient kv) {
setResourceGroupName(kv);
setAzureSubscriptionId(kv);
setEventGridTopicName(kv);
AzureTokenCredentials azureTokenCredentials = getAzureTokenCredentials();
return EventGridManager
.configure()
.withLogLevel(LogLevel.BASIC)
.authenticate(azureTokenCredentials, azureTokenCredentials.defaultSubscriptionId());
}
private AzureTokenCredentials getAzureTokenCredentials() {
AzureEnvironment azureEnvironment = new AzureEnvironment(new HashMap<>());
azureEnvironment.endpoints().putAll(AzureEnvironment.AZURE.endpoints());
return new ApplicationTokenCredentials(
azureClientId,
azureTenantId,
azureClientSecret,
azureEnvironment).withDefaultSubscriptionId(azureSubscriptionId);
}
String getKeyVaultSecret(SecretClient kv, String secretName) {
KeyVaultSecret secret = kv.getSecret(secretName);
if (secret == null) {
throw new IllegalStateException(String.format("No secret found with name %s", secretName));
}
@Value("${azure.keyvault.url}")
private String keyVaultURL;
String secretValue = secret.getValue();
if (secretValue == null) {
throw new IllegalStateException(String.format(
"Secret unexpectedly missing from KeyVault response for secret with name %s", secretName));
}
@Value("${azure.cosmosdb.database}")
private String cosmosDBName;
return secretValue;
}
@Value("${azure.appResourceId}")
private String azureAppResourceId;
@Autowired
@Named("COSMOS_ENDPOINT")
String cosmosEndpoint;
private String azureTenantId;
@Autowired
@Named("COSMOS_KEY")
String cosmosPrimaryKey;
@Bean
@Named("KEY_VAULT_URL")
public String keyVaultURL() {
return keyVaultURL;
}
@Bean
public CosmosClient buildCosmosClient(SecretClient kv) {
return new CosmosClientBuilder().endpoint(cosmosEndpoint).key(cosmosPrimaryKey).buildClient();
}
@Bean
@Named("COSMOS_DB_NAME")
public String cosmosDBName() {
return cosmosDBName;
}
@Inject
public void setAzureTenantId(@Named("APP_DEV_SP_TENANT_ID") String azureTenantId) {
this.azureTenantId = azureTenantId;
}
}
\ No newline at end of file
provider/register-azure/src/main/java/org/opengroup/osdu/register/provider/azure/di/SinglePartitionCosmosClientFactory.java deleted 100644 → 0
View file @ e7436b1d
package org.opengroup.osdu.register.provider.azure.di;
import com.azure.cosmos.CosmosClient;
import org.opengroup.osdu.azure.cosmosdb.ICosmosClientFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.annotation.Primary;
import org.springframework.stereotype.Component;
@Component
@Primary
public class SinglePartitionCosmosClientFactory implements ICosmosClientFactory {
@Autowired
CosmosClient cosmosClient;
@Override
public CosmosClient getClient(final String s) {
return cosmosClient;
}
}
provider/register-azure/src/main/java/org/opengroup/osdu/register/provider/azure/subscriber/PushSubscription.java
View file @ bbfbad5b
package org.opengroup.osdu.register.provider.azure.subscriber;
import com.microsoft.azure.CloudException;
import com.microsoft.azure.arm.model.Indexable;
import com.microsoft.azure.management.eventgrid.v2020_04_01_preview.WebHookEventSubscriptionDestination;
import com.microsoft.azure.management.eventgrid.v2020_04_01_preview.implementation.EventGridManager;
import java.util.concurrent.atomic.AtomicReference;
import org.opengroup.osdu.azure.eventgridmanager.EventGridManagerFactory;
import org.opengroup.osdu.core.common.logging.JaxRsDpsLog;
import org.opengroup.osdu.core.common.model.http.AppException;
import org.opengroup.osdu.core.common.model.http.DpsHeaders;
import org.opengroup.osdu.register.provider.azure.di.AzureBootstrapConfig;
import org.opengroup.osdu.register.provider.azure.util.PartitionHelper;
import org.opengroup.osdu.register.utils.AppServiceConfig;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
import java.util.concurrent.atomic.AtomicReference;
import com.microsoft.azure.CloudException;
import com.microsoft.azure.arm.model.Indexable;
import com.microsoft.azure.management.eventgrid.v2020_04_01_preview.WebHookEventSubscriptionDestination;
@Component
public class PushSubscription {
private static final String RESOURCE_PROVISIONING_ERROR_MESSAGE = "Resource cannot be updated during provisioning";
@Autowired
private EventGridManager eventGridManager;
@Autowired
private EventGridManagerFactory factory;
@Autowired
private DpsHeaders headers;
@Autowired
private AzureBootstrapConfig azureBootstrapConfig;
@Autowired
private JaxRsDpsLog logger;
@Autowired
private AppServiceConfig serviceConfig;
@Autowired
private PartitionHelper partitionHelper;
public void createPushSubscription(String subscriptionId, String topicName) {
WebHookEventSubscriptionDestination subscriptionDestination = new WebHookEventSubscriptionDestination();
......@@ -35,10 +40,10 @@ public class PushSubscription {
subscriptionDestination.withAzureActiveDirectoryTenantId(azureBootstrapConfig.getAzureTenantId());
subscriptionDestination.withAzureActiveDirectoryApplicationIdOrUri(azureBootstrapConfig.getAzureAppResourceId());
String scope = String.format("/subscriptions/%s/resourceGroups/%s/providers/Microsoft.EventGrid/topics/%s", azureBootstrapConfig.getAzureSubscriptionId(), azureBootstrapConfig.getResourceGroupName(), topicName);
String scope = partitionHelper.retrieveScope((topicName));
AtomicReference<Throwable> error = new AtomicReference<>();
eventGridManager.eventSubscriptions().define(subscriptionId)
factory.getManager(headers.getPartitionId()).eventSubscriptions().define(subscriptionId)
.withScope(scope)
.withDestination(subscriptionDestination)
.createAsync()
......@@ -63,9 +68,9 @@ public class PushSubscription {
}
public void deletePushSubscription(String subscriptionId, String topicName) {
String scope = String.format("/subscriptions/%s/resourceGroups/%s/providers/Microsoft.EventGrid/topics/%s", azureBootstrapConfig.getAzureSubscriptionId(), azureBootstrapConfig.getResourceGroupName(), topicName);
String scope = partitionHelper.retrieveScope(topicName);
AtomicReference<Throwable> error = new AtomicReference<>();
eventGridManager.eventSubscriptions().deleteAsync(scope, subscriptionId)
factory.getManager(headers.getPartitionId()).eventSubscriptions().deleteAsync(scope, subscriptionId)
.subscribe(() -> logger.info(String.format("Push Subscription with id %s deleted successfully", subscriptionId)), error::set);
if(error.get() != null) {
......@@ -82,9 +87,9 @@ public class PushSubscription {
}
public void checkIfPushSubscriptionExists(String subscriptionId, String topicName) {
String scope = String.format("/subscriptions/%s/resourceGroups/%s/providers/Microsoft.EventGrid/topics/%s", azureBootstrapConfig.getAzureSubscriptionId(), azureBootstrapConfig.getResourceGroupName(), topicName);
String scope = partitionHelper.retrieveScope(topicName);
AtomicReference<Throwable> error = new AtomicReference<>();
eventGridManager.eventSubscriptions().getAsync(scope, subscriptionId)
factory.getManager(headers.getPartitionId()).eventSubscriptions().getAsync(scope, subscriptionId)
.subscribe(
(Indexable indexable) -> {},
error::set,
......@@ -102,5 +107,4 @@ public class PushSubscription {
}
}
}
}
provider/register-azure/src/main/java/org/opengroup/osdu/register/provider/azure/subscriber/TopicsRepositoryImpl.java
View file @ bbfbad5b
package org.opengroup.osdu.register.provider.azure.subscriber;
import com.google.gson.Gson;
import com.google.gson.reflect.TypeToken;
import org.apache.http.HttpStatus;
import java.lang.reflect.Type;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import org.opengroup.osdu.core.common.logging.JaxRsDpsLog;
import org.opengroup.osdu.core.common.model.http.AppException;
import org.opengroup.osdu.core.common.model.http.DpsHeaders;
import org.opengroup.osdu.core.common.model.notification.Topic;
import org.opengroup.osdu.register.provider.azure.di.AzureBootstrapConfig;
import org.opengroup.osdu.register.provider.azure.util.PartitionHelper;
import org.opengroup.osdu.register.provider.interfaces.subscriber.ITopicsRepository;
import org.opengroup.osdu.register.utils.FileUtil;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.annotation.Primary;
import org.springframework.core.io.ClassPathResource;
import org.springframework.stereotype.Repository;
import java.io.ByteArrayOutputStream;
import java.io.InputStream;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.List;
import com.google.gson.Gson;
import com.google.gson.reflect.TypeToken;
@Repository
@Primary
public class TopicsRepositoryImpl implements ITopicsRepository {
@Autowired
private JaxRsDpsLog log;
@Autowired
private AzureBootstrapConfig config;
private DpsHeaders headers;
@Autowired
private PartitionHelper partitionHelper;
private List<Topic> topics;
private Map<String, List<Topic>> topicsMap = new HashMap<>();
public List<Topic> listMessages() {
if (topics == null) {
String topicName = config.getEventGridTopicName();
Gson gson = new Gson();
java.lang.reflect.Type listType = new TypeToken<ArrayList<Topic>>() {
}.getType();
topics = gson.fromJson(getFile(), listType);
Topic topic = topics.get(0);
topic.setName(topicName);
topics.set(0, topic);
String partitionId = headers.getPartitionId();
if (topicsMap.containsKey(partitionId)) {
return topicsMap.get(partitionId);
}
Type type = new TypeToken<ArrayList<Topic>>() {}.getType();
List<Topic> topics = new Gson().fromJson(FileUtil.readFile("topics.json", log), type);
topics.set(0, getTopic(topics));
topicsMap.put(partitionId, topics);
return topics;
}
private String getFile() {
final String fileName = "topics.json";
try {
InputStream inputStream = new ClassPathResource(fileName).getInputStream();
ByteArrayOutputStream result = new ByteArrayOutputStream();
byte[] buffer = new byte[1024];
int length;
while ((length = inputStream.read(buffer)) != -1) {
result.write(buffer, 0, length);
}
return result.toString(StandardCharsets.UTF_8.name());
} catch (Exception e) {
log.error("Error retrieving topics.json", e);
throw new AppException(HttpStatus.SC_SERVICE_UNAVAILABLE, "Server error", "An unexpected error occurred");
}
private Topic getTopic(List<Topic> topics) {
Topic topic = topics.get(0);
topic.setName(partitionHelper.retrieveTopicName());
return topic;
}
}
provider/register-azure/src/main/java/org/opengroup/osdu/register/provider/azure/util/CryptographyUtil.java
View file @ bbfbad5b
package org.opengroup.osdu.register.provider.azure.util;
import java.nio.charset.StandardCharsets;
import java.util.Base64;