Commit e9fd13ed authored by harshit aggarwal's avatar harshit aggarwal
Browse files

Refactoring Integration tests

parent 8a748417
......@@ -19,7 +19,11 @@
*.tar.gz
*.rar
### IntelliJ IDEA ###
.idea
*.iws
*.iml
*.ipr
### STS ###
target/
......
......@@ -42,6 +42,7 @@
<modules>
<module>wks-core</module>
<module>provider/wks-gcp</module>
<module>provider/wks-azure</module>
</modules>
<repositories>
......
## License
Copyright © Microsoft Corporation
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
### Environment Variables
In order to run the service locally, you will need to have the following environment variables defined.
**System Environment required to run service**
Refer to [application.properties](./src/main/resources/application.properties)
Definitions for some variables used
| name | value | description | sensitive? | source |
| --- | --- | --- | --- | --- |
| `LOG_PREFIX` | `storage` | Logging prefix | no | - |
| `server.servlet.contextPath` | `/api/storage/v2/` | Servlet context path | no | - |
| `STORAGE_API` | ex `https://foo-storage.azurewebsites.net` | Storage API endpoint | no | output of infrastructure deployment |
| `AUTHORIZE_API` | ex `https://foo-entitlements.azurewebsites.net` | Entitlements API endpoint | no | output of infrastructure deployment |
| `AUTHORIZE_API_KEY` | `********` | The API key clients will need to use when calling the entitlements | yes | -- |
| `azure.application-insights.instrumentation-key` | `********` | API Key for App Insights | yes | output of infrastructure deployment |
| `azure.activedirectory.client-id` | `********` | AAD client application ID | yes | output of infrastructure deployment |
| `azure.activedirectory.AppIdUri` | `api://${azure.activedirectory.client-id}` | URI for AAD Application | no | -- |
| `azure.activedirectory.session-stateless` | `true` | Flag run in stateless mode (needed by AAD dependency) | no | -- |
| `cosmosdb_account` | ex `devintosdur2cosmosacct` | Cosmos account name | no | output of infrastructure deployment |
| `cosmosdb_key` | `********` | Key for CosmosDB | yes | output of infrastructure deployments |
| `cosmosdb_database` | ex `dev-osdu-r2-db` | Cosmos database for storage documents | no | output of infrastructure deployment |
| `azure.storage.account-name` | ex `foo-storage-account` | Storage account for storing documents | no | output of infrastructure deployment |
| `azure.storage.enable-https` | `true` | Used by spring boot starter library | no | - |
| `servicebus_topic_name` | `recordstopic` | Topic for async messaging | no | output of infrastructure deployment |
| `servicebus_namespace_name` | ex `foo-sb-namespace` | Namespace for async messaging | no | output of infrastructure deployment |
| `KEYVAULT_URI` | ex `https://foo-keyvault.vault.azure.net/` | URI of KeyVault that holds application secrets | no | output of infrastructure deployment |
| `AZURE_CLIENT_ID` | `********` | Identity to run the service locally. This enables access to Azure resources. You only need this if running locally | yes | keyvault secret: `$KEYVAULT_URI/secrets/app-dev-sp-username` |
| `AZURE_TENANT_ID` | `********` | AD tenant to authenticate users from | yes | keyvault secret: `$KEYVAULT_URI/secrets/app-dev-sp-tenant-id` |
| `AZURE_CLIENT_SECRET` | `********` | Secret for `$AZURE_CLIENT_ID` | yes | keyvault secret: `$KEYVAULT_URI/secrets/app-dev-sp-password` |
### Build and run the application
After configuring your environment as specified above, you can follow these steps to build and run the application. These steps should be invoked from the *repository root*.
```bash
# build + test + install core service code
$ cd wks-core/ && mvn clean install
# build + test + package azure service code
$ cd provider/wks-azure/ && mvn clean package
# run service
#
# Note: this assumes that the environment variables for running the service as outlined
# above are already exported in your environment.
$ mvn spring-boot:run -Dspring-boot.run.profiles=local
# or directly run the jar file
$ cd provider/wks-azure/ && java -jar target\wks-azure-1.0.0-spring-boot.jar
```
\ No newline at end of file
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<parent>
<artifactId>os-wks</artifactId>
<groupId>org.opengroup.osdu</groupId>
<version>0.0.1</version>
<relativePath>../../pom.xml</relativePath>
</parent>
<modelVersion>4.0.0</modelVersion>
<artifactId>wks-azure</artifactId>
<description>WKS service on Azure</description>
<packaging>jar</packaging>
<properties>
<azure.version>2.1.7</azure.version>
</properties>
<dependencies>
<dependency>
<groupId>com.microsoft.azure</groupId>
<artifactId>azure-active-directory-spring-boot-starter</artifactId>
<version>${azure.version}</version>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter</artifactId>
<exclusions>
<exclusion>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-logging</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-security</artifactId>
<exclusions>
<exclusion>
<groupId>ch.qos.logback</groupId>
<artifactId>logback-classic</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-to-slf4j</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.opengroup.osdu</groupId>
<artifactId>os-wks-core</artifactId>
<version>0.0.1</version>
</dependency>
<dependency>
<groupId>org.opengroup.osdu</groupId>
<artifactId>os-core-common</artifactId>
<version>0.0.11</version>
</dependency>
<dependency>
<groupId>org.opengroup.osdu</groupId>
<artifactId>core-lib-azure</artifactId>
<version>0.0.17</version>
</dependency>
<dependency>
<groupId>org.projectlombok</groupId>
<artifactId>lombok</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-test</artifactId>
<scope>test</scope>
</dependency>
<!--
Override the spring-boot version of these dependencies to the ones
required by the azure-core library. This needs to be done for each
app that depends on this library
-->
<dependency>
<groupId>io.projectreactor.netty</groupId>
<artifactId>reactor-netty</artifactId>
<version>0.9.0.RELEASE</version>
</dependency>
<dependency>
<groupId>io.projectreactor</groupId>
<artifactId>reactor-core</artifactId>
<version>3.3.0.RELEASE</version>
</dependency>
<dependency>
<groupId>org.junit.jupiter</groupId>
<artifactId>junit-jupiter</artifactId>
<version>5.6.0</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.mockito</groupId>
<artifactId>mockito-junit-jupiter</artifactId>
<version>2.23.0</version>
<scope>test</scope>
</dependency>
</dependencies>
<repositories>
<repository>
<id>${gitlab-server}</id>
<url>https://community.opengroup.org/api/v4/groups/17/-/packages/maven</url>
</repository>
</repositories>
<distributionManagement>
<repository>
<id>${gitlab-server}</id>
<url>https://community.opengroup.org/api/v4/projects/44/packages/maven</url>
</repository>
<snapshotRepository>
<id>${gitlab-server}</id>
<url>https://community.opengroup.org/api/v4/projects/44/packages/maven</url>
</snapshotRepository>
</distributionManagement>
<build>
<plugins>
<plugin>
<artifactId>maven-surefire-plugin</artifactId>
<version>2.22.2</version>
</plugin>
<plugin>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-maven-plugin</artifactId>
<executions>
<execution>
<goals>
<goal>repackage</goal>
</goals>
<configuration>
<classifier>spring-boot</classifier>
<mainClass>org.opengroup.osdu.wks.WksServiceApplication</mainClass>
</configuration>
</execution>
</executions>
</plugin>
</plugins>
</build>
</project>
package org.opengroup.osdu.wks.provider.azure.credentials;
import org.opengroup.osdu.azure.util.AzureServicePrincipal;
import org.opengroup.osdu.core.common.model.tenant.TenantInfo;
import org.opengroup.osdu.wks.provider.azure.di.AzureBootstrapConfig;
import org.opengroup.osdu.wks.provider.interfaces.UserCredential;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.annotation.Configuration;
@Configuration
public class JwtTokenGenerator implements UserCredential {
private static final String BEARER = "Bearer ";
@Autowired
private AzureBootstrapConfig azureBootstrapConfig;
private AzureServicePrincipal azureServicePrincipal;
@Override
public String getIdToken(TenantInfo tenant) {
if(azureServicePrincipal == null) {
azureServicePrincipal = new AzureServicePrincipal();
}
String token = null;
try {
token = BEARER + azureServicePrincipal.getIdToken(
azureBootstrapConfig.getClientId(),
azureBootstrapConfig.getClientSecret(),
azureBootstrapConfig.getTenantId(),
azureBootstrapConfig.getAppResourceId());
}
catch (Exception e) {
System.out.println(e.getMessage());
}
return token;
}
}
package org.opengroup.osdu.wks.provider.azure.di;
import com.azure.security.keyvault.secrets.SecretClient;
import com.azure.security.keyvault.secrets.models.KeyVaultSecret;
import com.microsoft.azure.servicebus.ReceiveMode;
import com.microsoft.azure.servicebus.SubscriptionClient;
import com.microsoft.azure.servicebus.primitives.ConnectionStringBuilder;
import com.microsoft.azure.servicebus.primitives.ServiceBusException;
import lombok.Getter;
import org.opengroup.osdu.core.common.model.http.AppException;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import javax.inject.Named;
@Configuration
@Getter
public class AzureBootstrapConfig {
@Value("${azure.storage.account-name}")
private String storageAccount;
@Value("${azure.storage.container-name}")
private String storageContainer;
@Value("${azure.servicebus.topic-name}")
private String serviceBusTopic;
@Value("${azure.servicebus.connection-string}")
private String serviceBusConnectionString;
@Value("${azure.servicebus.topic-subscription}")
private String serviceBusTopicSubscription;
@Value("${azure.keyvault.url}")
private String keyVaultURL;
@Value("${azure.cosmosdb.database}")
private String cosmosDBName;
@Value("${azure-client-id}")
private String clientId;
@Value("${azure-client-secret}")
private String clientSecret;
@Value("${azure-tenant-id}")
private String tenantId;
@Value("${azure-app-resource-id}")
private String appResourceId;
@Value("${executor-n-threads}")
private String nThreads;
@Value("${max_concurrent_calls}")
private String maxConcurrentCalls;
@Bean
@Named("STORAGE_ACCOUNT_NAME")
public String storageAccount() {
return storageAccount;
}
@Bean
@Named("STORAGE_CONTAINER_NAME")
public String containerName() {
return storageContainer;
}
@Bean
@Named("KEY_VAULT_URL")
public String keyVaultURL() {
return keyVaultURL;
}
@Bean
@Named("COSMOS_DB_NAME")
public String cosmosDBName() {
return cosmosDBName;
}
@Bean
@Named("COSMOS_ENDPOINT")
public String cosmosEndpoint(SecretClient kv) {
return getKeyVaultSecret(kv, "cosmos-endpoint");
}
@Bean
@Named("COSMOS_KEY")
public String cosmosKey(SecretClient kv) {
return getKeyVaultSecret(kv, "cosmos-primary-key");
}
@Bean
@Named("SUBSCRIPTION_CLIENT")
public SubscriptionClient subscriptionClient() {
String entityPath = serviceBusTopic + "/subscriptions/" + serviceBusTopicSubscription;
ConnectionStringBuilder connectionStringBuilder = new ConnectionStringBuilder(
serviceBusConnectionString,
entityPath
);
SubscriptionClient subscriptionClient;
try {
subscriptionClient = new SubscriptionClient(connectionStringBuilder, ReceiveMode.PEEKLOCK);
} catch (InterruptedException | ServiceBusException e) {
throw new AppException(500, "Server Error", "Unexpected error creating Subscription Client", e);
}
return subscriptionClient;
}
String getKeyVaultSecret(SecretClient kv, String secretName) {
KeyVaultSecret secret = kv.getSecret(secretName);
if (secret == null) {
throw new IllegalStateException(String.format("No secret found with name %s", secretName));
}
String secretValue = secret.getValue();
if (secretValue == null) {
throw new IllegalStateException(String.format(
"Secret unexpectedly missing from KeyVault response for secret with name %s", secretName));
}
return secretValue;
}
}
package org.opengroup.osdu.wks.provider.azure.pubsub;
import com.microsoft.azure.servicebus.ExceptionPhase;
import com.microsoft.azure.servicebus.IMessage;
import com.microsoft.azure.servicebus.IMessageHandler;
import com.microsoft.azure.servicebus.SubscriptionClient;
import lombok.extern.java.Log;
import java.util.concurrent.CompletableFuture;
@Log
public class MessageHandler implements IMessageHandler {
private final SubscriptionClient receiveClient;
private final ProcessWKSTransform processWKSTransform;
public MessageHandler(SubscriptionClient client, ProcessWKSTransform processWKSTransform) {
this.receiveClient = client;
this.processWKSTransform = processWKSTransform;
}
@Override
public CompletableFuture<Void> onMessageAsync(IMessage message) {
boolean ack = this.processWKSTransform.initiateWksTransformation(message);
if(ack) {
return receiveClient.completeAsync(message.getLockToken());
}
else {
return receiveClient.abandonAsync(message.getLockToken());
}
}
@Override
public void notifyException(Throwable throwable, ExceptionPhase exceptionPhase) {
log.severe(exceptionPhase + "-" + throwable.getMessage());
}
}
package org.opengroup.osdu.wks.provider.azure.pubsub;
import com.google.gson.Gson;
import com.google.gson.JsonElement;
import com.google.gson.JsonParser;
import com.microsoft.azure.servicebus.IMessage;
import lombok.extern.java.Log;
import org.opengroup.osdu.core.common.model.http.DpsHeaders;
import org.opengroup.osdu.wks.exceptions.ApplicationException;
import org.opengroup.osdu.wks.exceptions.BadRequestException;
import org.opengroup.osdu.wks.model.RawRecordDetails;
import org.opengroup.osdu.wks.service.WKSService;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.http.HttpStatus;
import org.springframework.stereotype.Component;
import java.util.logging.Level;
import static java.nio.charset.StandardCharsets.UTF_8;
@Component
@Log
public class ProcessWKSTransform {
private static final String DATA = "data";
@Autowired
private WKSService wKSService;
public boolean initiateWksTransformation(IMessage message) {
String dataPartitionId = message.getProperties().get(DpsHeaders.DATA_PARTITION_ID).toString();
String correlationId = message.getProperties().get(DpsHeaders.CORRELATION_ID).toString();
try {
RawRecordDetails[] rawRecordDetails = retrieveDataFromMessage(message);
wKSService.transform(rawRecordDetails, dataPartitionId, correlationId);
} catch (BadRequestException e) {
log.severe(e.getErrorMsg());
log.log(Level.SEVERE, String.format("Bad Request Reason: %s, pubsub message id: %s", e.getErrorMsg(),
message.getMessageId()));
} catch (ApplicationException e) {
log.severe(e.getErrorMsg());
log.log(Level.SEVERE, String.format("Application Error Reason: %s, pubsub message id: %s", e.getErrorMsg(),
message.getMessageId()));
return false;
}
return true;
}
private RawRecordDetails[] retrieveDataFromMessage(IMessage message) throws BadRequestException {
String messageData = new String(message.getMessageBody().getBinaryData().get(0), UTF_8);
JsonElement jsonRoot = JsonParser.parseString(messageData);
JsonElement msg = jsonRoot.getAsJsonObject().get("message");
if (msg == null) {
throw new BadRequestException(HttpStatus.BAD_REQUEST,"Invalid record change message, message object not found");
}
String dataValue = msg.getAsJsonObject().get(DATA).toString();
return getRecordsChangeData(dataValue);
}
private RawRecordDetails[] getRecordsChangeData(String data) throws BadRequestException {
if (data.equals("[]")) {
throw new BadRequestException(HttpStatus.BAD_REQUEST,"Invalid record change message, message data not found");
}
Gson gson = new Gson();
return gson.fromJson(data, RawRecordDetails[].class);
}
}
package org.opengroup.osdu.wks.provider.azure.pubsub;
import com.microsoft.azure.servicebus.SubscriptionClient;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
@Component
public class SubscriptionClientFactory {
@Autowired
private SubscriptionClient subscriptionClient;
public SubscriptionClient getSubscriptionClient() {
return subscriptionClient;
}
}
package org.opengroup.osdu.wks.provider.azure.pubsub;
import com.microsoft.azure.servicebus.MessageHandlerOptions;
import com.microsoft.azure.servicebus.SubscriptionClient;
import com.microsoft.azure.servicebus.primitives.ServiceBusException;
import lombok.extern.java.Log;
import org.opengroup.osdu.core.common.model.tenant.TenantInfo;
import org.opengroup.osdu.core.common.provider.interfaces.ITenantFactory;
import org.opengroup.osdu.wks.provider.azure.di.AzureBootstrapConfig;
import org.opengroup.osdu.wks.provider.interfaces.SubscriptionManager;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
import java.time.Duration;
import java.util.Collection;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.logging.Level;
@Component
@Log
public class SubscriptionManagerImpl implements SubscriptionManager {
@Autowired
private ITenantFactory tenantFactory;
@Autowired
private SubscriptionClientFactory subscriptionClientFactory;
@Autowired
private ProcessWKSTransform processWKSTransform;
@Autowired
private AzureBootstrapConfig azureBootstrapConfig;
@Override
public void subscribeRecordsChangeEvent() {
// TODO subscribe for every tenant currently subscribing for opendes
SubscriptionClient subscriptionClient = this.subscriptionClientFactory.getSubscriptionClient();