Skip to content
GitLab
Commit 88647b22 authored by Anuj Gupta's avatar Anuj Gupta
Browse files

Merge branch 'ibm-int-impl' into 'master' 

ibm-int-impl

See merge request !122
parents f11ec111 5d330687
Pipeline #67162 failed with stages
in 23 minutes and 31 seconds
Hide whitespace changes
Inline Side-by-side
.gitlab-ci.yml
View file @ 88647b22
......@@ -7,6 +7,7 @@ variables:
AWS_EKS_DEPLOYMENT_NAME: os-dataset
IBM_BUILD_SUBDIR: provider/dataset-ibm
IBM_INT_TEST_SUBDIR: testing/dataset-test-ibm
# --- osdu-gcp specific variables ---
OSDU_GCP_SERVICE: dataset
......@@ -42,7 +43,7 @@ include:
file: 'cloud-providers/azure.yml'
- project: "osdu/platform/ci-cd-pipelines"
file: "cloud-providers/ibm-deploy-only.yml"
file: "cloud-providers/ibm.yml"
- project: "osdu/platform/ci-cd-pipelines"
file: "scanners/gitlab-ultimate.yml"
......
provider/dataset-ibm/pom.xml
View file @ 88647b22
<?xml version="1.0" encoding="UTF-8"?>
<!--
Copyright © 2020 Amazon Web Services
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<parent>
......
provider/dataset-ibm/src/main/java/org/opengroup/osdu/dataset/provider/ibm/DatasetDmsServiceMapImpl.java
View file @ 88647b22
......@@ -8,7 +8,6 @@ import java.util.Map;
import javax.annotation.PostConstruct;
import org.apache.commons.lang3.StringUtils;
import org.opengroup.osdu.dataset.dms.DmsServiceProperties;
import org.opengroup.osdu.dataset.provider.interfaces.IDatasetDmsServiceMap;
import org.springframework.beans.factory.annotation.Value;
......@@ -17,25 +16,20 @@ import org.springframework.stereotype.Service;
@Service
public class DatasetDmsServiceMapImpl implements IDatasetDmsServiceMap {
@Value("${DMS_API_BASE}")
private String DMS_API_BASE;
private Map<String,DmsServiceProperties> resourceTypeToDmsServiceMap = new HashMap<>();
private Map<String,DmsServiceProperties> resourceTypeToDmsServiceMap = new HashMap<>();
@Value("${FILE_API}")
private String fileApi;
@PostConstruct
public void init() {
//todo: replace this with service discovery / registered db entries
resourceTypeToDmsServiceMap.put(
"dataset--File.*",
new DmsServiceProperties(StringUtils.join(DMS_API_BASE, "/api/dms/file/v1/file"))
);
resourceTypeToDmsServiceMap.put(
"dataset--FileCollection.*",
new DmsServiceProperties(StringUtils.join(DMS_API_BASE, "/api/dms/file/v1/file-collection"))
);
}
public void init() {
DmsServiceProperties fileDmsProperties = new DmsServiceProperties(fileApi);
fileDmsProperties.setStagingLocationSupported(true);
resourceTypeToDmsServiceMap.put("dataset--File.*", fileDmsProperties);
resourceTypeToDmsServiceMap.put("dataset--FileCollection.*",fileDmsProperties);
}
@Override
public Map<String, DmsServiceProperties> getResourceTypeToDmsServiceMap() {
......
testing/dataset-test-ibm/pom.xml 0 → 100644
View file @ 88647b22
<?xml version="1.0" encoding="UTF-8"?>
<!-- Copyright 2020 IBM Corp. All Rights Reserved. Licensed under the Apache
License, Version 2.0 (the "License"); you may not use this file except in
compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software distributed
under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES
OR CONDITIONS OF ANY KIND, either express or implied. See the License for
the specific language governing permissions and limitations under the License. -->
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.opengroup.osdu</groupId>
<artifactId>os-dataset-testing</artifactId>
<version>0.12.0-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
<artifactId>dataset-test-ibm</artifactId>
<packaging>jar</packaging>
<properties>
<maven.compiler.target>1.8</maven.compiler.target>
<maven.compiler.source>1.8</maven.compiler.source>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<project.main.basedir>${project.basedir}</project.main.basedir>
<os-core-common.version>0.11.0</os-core-common.version>
<os-core-lib-ibm.version>0.9.0</os-core-lib-ibm.version>
</properties>
<dependencies>
<dependency>
<groupId>org.opengroup.osdu.storage</groupId>
<artifactId>dataset-test-core</artifactId>
<version>0.12.0-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>org.opengroup.osdu</groupId>
<artifactId>os-core-common</artifactId>
<version>0.11.0</version>
</dependency>
<dependency>
<groupId>org.opengroup.osdu</groupId>
<artifactId>os-core-lib-ibm</artifactId>
<version>${os-core-lib-ibm.version}</version>
</dependency>
<dependency>
<groupId>com.sun.jersey</groupId>
<artifactId>jersey-client</artifactId>
<version>1.19.4</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<version>4.12</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.opengroup.osdu</groupId>
<artifactId>os-core-common</artifactId>
<version>${os-core-common.version}</version>
<exclusions>
<exclusion>
<groupId>io.netty</groupId>
<artifactId>netty-common</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.apache.httpcomponents</groupId>
<artifactId>httpclient</artifactId>
<version>4.5.2</version>
<scope>runtime</scope>
</dependency>
<dependency>
<groupId>io.netty</groupId>
<artifactId>netty-all</artifactId>
<version>4.1.50.Final</version>
</dependency>
<dependency>
<groupId>io.netty</groupId>
<artifactId>netty-transport</artifactId>
<version>4.1.15.Final</version>
</dependency>
</dependencies>
<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-surefire-plugin</artifactId>
<version>2.5</version>
<executions>
<execution>
<phase>integration-test</phase>
<goals>
<goal>test</goal>
</goals>
<configuration>
<excludes>
<exclude>none</exclude>
</excludes>
</configuration>
</execution>
</executions>
<configuration>
<trimStackTrace>false</trimStackTrace>
<systemPropertyVariables>
<buildDirectory>${project.build.directory}</buildDirectory>
</systemPropertyVariables>
</configuration>
</plugin>
</plugins>
</build>
</project>
testing/dataset-test-ibm/src/test/java/org/opengroup/osdu/dataset/ibm/TestDatasetIbm.java 0 → 100644
View file @ 88647b22
/* Licensed Materials - Property of IBM */
/* (c) Copyright IBM Corp. 2020. All Rights Reserved.*/
package org.opengroup.osdu.dataset.ibm;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
import org.junit.AfterClass;
import org.junit.Assert;
import org.junit.Before;
import org.junit.BeforeClass;
import org.opengroup.osdu.core.common.dms.model.RetrievalInstructionsResponse;
import org.opengroup.osdu.core.common.dms.model.StorageInstructionsResponse;
import org.opengroup.osdu.dataset.DatasetIT;
import org.opengroup.osdu.dataset.ibm.util.CloudStorageUtilIbm;
import org.opengroup.osdu.dataset.ibm.util.IBMTestUtils;
public class TestDatasetIbm extends DatasetIT {
private static final IBMTestUtils ibmTestUtils = new IBMTestUtils();
public TestDatasetIbm() {
cloudStorageUtil = new CloudStorageUtilIbm();
}
@BeforeClass
public static void classSetup() throws Exception {
DatasetIT.classSetup(ibmTestUtils.getToken());
}
@AfterClass
public static void classTearDown() throws Exception {
DatasetIT.classTearDown(ibmTestUtils.getToken());
}
@Before
@Override
public void setup() throws Exception {
runTests = true;
testUtils = ibmTestUtils;
}
@Override
public void tearDown() throws Exception {
}
@Override
public void validateStorageInstructions(StorageInstructionsResponse storageInstructions) {
assertEquals("IBM", storageInstructions.getProviderKey());
assertTrue(storageInstructions.getStorageLocation().containsKey("signedUrl"));
assertTrue(storageInstructions.getStorageLocation().containsKey("fileSource"));
}
@Override
public void validateRetrievalInstructions(RetrievalInstructionsResponse retrievalInstructions,
int expectedDatasets) {
assertEquals("IBM", retrievalInstructions.getProviderKey());
assertEquals(expectedDatasets, retrievalInstructions.getDatasets().size());
}
}
testing/dataset-test-ibm/src/test/java/org/opengroup/osdu/dataset/ibm/util/CloudStorageUtilIbm.java 0 → 100644
View file @ 88647b22
/* Licensed Materials - Property of IBM */
/* (c) Copyright IBM Corp. 2020. All Rights Reserved.*/
package org.opengroup.osdu.dataset.ibm.util;
import java.util.HashMap;
import java.util.Map;
import org.opengroup.osdu.dataset.CloudStorageUtil;
import org.opengroup.osdu.dataset.TestUtils;
import com.sun.jersey.api.client.ClientResponse;
public class CloudStorageUtilIbm extends CloudStorageUtil {
@Override
public String uploadCloudFileUsingProvidedCredentials(String fileName, Object storageLocationProperties,
String fileContents) throws Exception {
// Upload File to Signed URL
Map<String, String> fileUploadHeaders = new HashMap<>();
fileUploadHeaders.put("x-ms-blob-type", "BlockBlob");
Map<String, Object> storageLocation = (Map<String, Object>) storageLocationProperties;
ClientResponse fileUploadResponse = TestUtils.send(
storageLocation.get("signedUrl").toString(), "",
"PUT", fileUploadHeaders, fileContents, null);
return storageLocation.get("fileSource").toString();
}
@Override
public String downloadCloudFileUsingDeliveryItem(Object retrievalLocationProperties) throws Exception {
Map<String, Object> retrievalProperties = (Map<String, Object>) retrievalLocationProperties;
ClientResponse fileUploadResponse = TestUtils.send(
retrievalProperties.get("signedUrl").toString(), "",
"GET", new HashMap<>(), "", null);
String downloadedFileResp = null;
downloadedFileResp = fileUploadResponse.getEntity(String.class);
return downloadedFileResp;
}
@Override
public void deleteCloudFile(String unsignedUrl) {
}
}
\ No newline at end of file
testing/dataset-test-ibm/src/test/java/org/opengroup/osdu/dataset/ibm/util/IBMTestUtils.java 0 → 100644
View file @ 88647b22
/* Licensed Materials - Property of IBM */
/* (c) Copyright IBM Corp. 2020. All Rights Reserved.*/
package org.opengroup.osdu.dataset.ibm.util;
import org.opengroup.osdu.core.ibm.util.IdentityClient;
import org.opengroup.osdu.dataset.TestUtils;
import com.google.common.base.Strings;
public class IBMTestUtils extends TestUtils {
private static String token;
private static String noDataAccesstoken;
@Override
public synchronized String getToken() throws Exception {
if (Strings.isNullOrEmpty(token)) {
token = IdentityClient.getTokenForUserWithAccess();
}
return "Bearer " + token;
}
@Override
public synchronized String getNoDataAccessToken() throws Exception {
if (Strings.isNullOrEmpty(noDataAccesstoken)) {
noDataAccesstoken = IdentityClient.getTokenForUserWithNoAccess();
}
return "Bearer " + noDataAccesstoken;
}
}
testing/pom.xml
View file @ 88647b22
......@@ -35,6 +35,7 @@
<module>dataset-test-azure</module>
<module>dataset-test-aws</module>
<module>dataset-test-gcp</module>
<module>dataset-test-ibm</module>
</modules>
<profiles>
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment