Skip to content

GitLab

Commit aa666876 authored by Alan Braz's avatar Alan Braz
Browse files

Merge branch 'ibm-impl' into 'master' 

Ibm impl

See merge request !1
parents 227d0536 0f19baa3
Pipeline #5025 passed with stages
in 9 minutes and 55 seconds
Hide whitespace changes
Inline Side-by-side
.gitignore
View file @ aa666876
......@@ -39,4 +39,7 @@ buildNumber.properties
### macOS ###
# General
.DS_Store
\ No newline at end of file
.DS_Store
.project
.settings
.classpath
.gitlab-ci.yml
View file @ aa666876
variables:
IBM_BUILD_SUBDIR: provider/file-ibm
#IBM_INT_TEST_SUBDIR: testing/file-test-ibm
include:
- project: 'osdu/platform/ci-cd-pipelines'
ref: 'master'
......@@ -7,14 +11,18 @@ include:
ref: 'master'
file: 'build/maven.yml'
- project: 'osdu/platform/ci-cd-pipelines'
ref: 'master'
file: 'scanners/gitlab-ultimate.yml'
# - project: 'osdu/platform/ci-cd-pipelines'
# ref: 'master'
# file: 'scanners/gitlab-ultimate.yml'
#
# - project: 'osdu/platform/ci-cd-pipelines'
# ref: 'master'
# file: 'scanners/fossa.yml'
#
# - project: 'osdu/platform/ci-cd-pipelines'
# ref: 'master'
# file: 'publishing/pages.yml'
- project: 'osdu/platform/ci-cd-pipelines'
ref: 'master'
file: 'scanners/fossa.yml'
- project: 'osdu/platform/ci-cd-pipelines'
- project: "osdu/platform/ci-cd-pipelines"
ref: 'master'
file: 'publishing/pages.yml'
file: "cloud-providers/ibm.yml"
pom.xml
View file @ aa666876
......@@ -40,6 +40,7 @@
<module>testing/file-test-core</module>
<!-- <module>provider/file-aws</module> FIX: Missing dependencies -->
<module>provider/file-gcp</module>
<module>provider/file-ibm</module>
<module>provider/file-gcp-datastore</module>
</modules>
......
provider/file-ibm/pom.xml 0 → 100644
View file @ aa666876
<?xml version="1.0" encoding="UTF-8" ?>
<!-- ~ ~ Licensed under the Apache License, Version 2.0 (the "License");
~ you may not use this file except in compliance with the License. ~ You
may obtain a copy of the License at ~ ~ https://www.apache.org/licenses/LICENSE-2.0
~ ~ Unless required by applicable law or agreed to in writing, software ~
distributed under the License is distributed on an "AS IS" BASIS, ~ WITHOUT
WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. ~ See the
License for the specific language governing permissions and ~ limitations
under the License. -->
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.opengroup.osdu</groupId>
<artifactId>os-file</artifactId>
<version>0.0.1-SNAPSHOT</version>
<relativePath>../../pom.xml</relativePath>
</parent>
<artifactId>file-ibm</artifactId>
<packaging>jar</packaging>
<name>file-ibm</name>
<description>IBM implementation of File service APIs</description>
<properties>
<start-class>org.opengroup.osdu.file.provider.ibm.FileIBMApplication</start-class>
<os-core-lib-ibm.version>0.0.19-SNAPSHOT</os-core-lib-ibm.version>
</properties>
<dependencies>
<dependency>
<groupId>org.opengroup.osdu</groupId>
<artifactId>file-core</artifactId>
<version>0.0.1-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>org.opengroup.osdu</groupId>
<artifactId>os-core-lib-ibm</artifactId>
<version>${os-core-lib-ibm.version}</version>
</dependency>
<!-- Test Dependencies -->
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<scope>test</scope>
</dependency>
<!-- https://mvnrepository.com/artifact/org.powermock/powermock-api-mockito2 -->
<dependency>
<groupId>org.powermock</groupId>
<artifactId>powermock-api-mockito2</artifactId>
<version>2.0.2</version>
<scope>test</scope>
</dependency>
<!-- https://mvnrepository.com/artifact/org.powermock/powermock-module-junit4 -->
<dependency>
<groupId>org.powermock</groupId>
<artifactId>powermock-module-junit4</artifactId>
<version>2.0.2</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.mockito</groupId>
<artifactId>mockito-core</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.codehaus.groovy</groupId>
<artifactId>groovy-jsr223</artifactId>
<version>2.5.8</version>
<type>pom</type>
<scope>test</scope>
</dependency>
</dependencies>
<repositories>
<repository>
<id>${gitlab-server}</id>
<url>https://community.opengroup.org/api/v4/groups/17/-/packages/maven</url>
</repository>
</repositories>
<distributionManagement>
<repository>
<id>${gitlab-server}</id>
<url>https://community.opengroup.org/api/v4/projects/44/packages/maven</url>
</repository>
<snapshotRepository>
<id>${gitlab-server}</id>
<url>https://community.opengroup.org/api/v4/projects/44/packages/maven</url>
</snapshotRepository>
</distributionManagement>
<build>
<plugins>
<plugin>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-maven-plugin</artifactId>
<configuration>
<mainClass>${start-class}</mainClass>
</configuration>
<executions>
<execution>
<goals>
<goal>repackage</goal>
</goals>
<configuration>
<classifier>spring-boot</classifier>
<mainClass>${start-class}</mainClass>
</configuration>
</execution>
</executions>
</plugin>
</plugins>
</build>
</project>
provider/file-ibm/src/main/java/org/opengroup/osdu/file/provider/ibm/FileIBMApplication.java 0 → 100644
View file @ aa666876
/*
* Copyright 2020 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.opengroup.osdu.file.provider.ibm;
import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication;
@SpringBootApplication(scanBasePackages = "org.opengroup.osdu")
public class FileIBMApplication {
public static void main(String[] args) {
//BasicConfigurator.configure();
SpringApplication.run(FileIBMApplication.class, args);
}
}
provider/file-ibm/src/main/java/org/opengroup/osdu/file/provider/ibm/mapper/IBMLocationMapper.java 0 → 100644
View file @ aa666876
/*
* Copyright 2020 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.opengroup.osdu.file.provider.ibm.mapper;
import java.util.HashMap;
import java.util.Map;
import org.opengroup.osdu.core.common.model.file.FileLocation;
import org.opengroup.osdu.core.common.model.file.LocationResponse;
import org.opengroup.osdu.file.model.SignedUrl;
import org.opengroup.osdu.file.provider.interfaces.ILocationMapper;
import org.springframework.stereotype.Component;
@Component
public class IBMLocationMapper implements ILocationMapper {
private static final String SIGNED_URL_KEY = "SignedURL";
@Override
public LocationResponse buildLocationResponse(SignedUrl signedUrl, FileLocation fileLocation) {
Map<String, String> location = new HashMap<>();
location.put(SIGNED_URL_KEY, signedUrl.getUrl().toString());
return LocationResponse.builder()
.fileID(fileLocation.getFileID())
.location(location)
.build();
}
}
provider/file-ibm/src/main/java/org/opengroup/osdu/file/provider/ibm/model/constant/StorageConstant.java 0 → 100644
View file @ aa666876
/*
* Copyright 2020 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.opengroup.osdu.file.provider.ibm.model.constant;
public final class StorageConstant {
public static final String S3_PROTOCOL = "s3://";
public static final int GCS_MAX_FILEPATH = 1024;
private StorageConstant() {
}
}
provider/file-ibm/src/main/java/org/opengroup/osdu/file/provider/ibm/model/file/FileLocationDoc.java 0 → 100644
View file @ aa666876
/*
* Copyright 2020 Google LLC
* Copyright 2017-2019, Schlumberger
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.opengroup.osdu.file.provider.ibm.model.file;
import java.util.Date;
import org.opengroup.osdu.core.common.model.file.FileLocation;
import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.NoArgsConstructor;
@Data
@NoArgsConstructor
@AllArgsConstructor
public class FileLocationDoc extends FileLocation {
String _id;
String _rev;
Long createdDate;
public FileLocationDoc(FileLocation fileLocation) {
this._id = fileLocation.getFileID();
this.setCreatedDate(fileLocation.getCreatedAt().getTime());
this.setCreatedBy(fileLocation.getCreatedBy());
this.setDriver(fileLocation.getDriver());
this.setLocation(fileLocation.getLocation());
}
public FileLocation getFileLocation() {
return new FileLocation(_id, super.getDriver(), super.getLocation(), new Date(createdDate), super.getCreatedBy());
}
}
provider/file-ibm/src/main/java/org/opengroup/osdu/file/provider/ibm/repository/IBMFileRepositoryImpl.java 0 → 100644
View file @ aa666876
/**
* Copyright 2020 IBM Corp. All Rights Reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.opengroup.osdu.file.provider.ibm.repository;
import static com.cloudant.client.api.query.Expression.eq;
import static com.cloudant.client.api.query.Expression.gte;
import static com.cloudant.client.api.query.Expression.lte;
import static com.cloudant.client.api.query.Operation.and;
import static java.lang.String.format;
import java.net.MalformedURLException;
import java.time.LocalDateTime;
import java.time.ZoneOffset;
import java.util.Date;
import java.util.List;
import java.util.stream.Collectors;
import javax.annotation.PostConstruct;
import org.apache.http.HttpStatus;
import org.opengroup.osdu.core.common.model.file.FileListRequest;
import org.opengroup.osdu.core.common.model.file.FileListResponse;
import org.opengroup.osdu.core.common.model.file.FileLocation;
import org.opengroup.osdu.core.ibm.auth.ServiceCredentials;
import org.opengroup.osdu.core.ibm.cloudant.IBMCloudantClientFactory;
import org.opengroup.osdu.file.exception.FileLocationNotFoundException;
import org.opengroup.osdu.file.provider.ibm.model.file.FileLocationDoc;
import org.opengroup.osdu.file.provider.interfaces.IFileLocationRepository;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Repository;
import com.cloudant.client.api.Database;
import com.cloudant.client.api.model.Response;
import com.cloudant.client.api.query.JsonIndex;
import com.cloudant.client.api.query.QueryBuilder;
import com.cloudant.client.api.query.QueryResult;
import com.cloudant.client.org.lightcouch.NoDocumentException;
import lombok.extern.java.Log;
@Repository
@Log
public class IBMFileRepositoryImpl implements IFileLocationRepository {
@Value("${ibm.db.url}")
private String dbUrl;
@Value("${ibm.db.user:#{null}}")
private String dbUser;
@Value("${ibm.db.password:#{null}}")
private String dbPassword;
@Value("${ibm.env.prefix:local-dev}")
private String dbNamePrefix;
@Value("${ibm.schemaName:file-locations}")
public String DB_NAME;
private Database db;
@PostConstruct
public void init() throws MalformedURLException {
IBMCloudantClientFactory cloudantFactory = new IBMCloudantClientFactory(
new ServiceCredentials(dbUrl, dbUser, dbPassword));
db = cloudantFactory.getDatabase(dbNamePrefix, DB_NAME);
db.createIndex(JsonIndex.builder().name("find-json-index").asc("createdDate", "createdBy").definition());
}
@Override
public FileLocation findByFileID(String fileID) {
log.info("Requesting file location. File ID : " + fileID);
try {
FileLocationDoc doc = db.find(FileLocationDoc.class, fileID);
return doc.getFileLocation();
} catch (NoDocumentException e) {
return null;
}
}
@Override
public FileLocation save(FileLocation fileLocation) {
String fileName = fileLocation.getFileID();
if (db.contains(fileName)) {
log.severe("File " + fileName + " already exist. Can't create again.");
throw new IllegalArgumentException("File " + fileName + " already exist. Can't create again.");
}
FileLocationDoc doc = new FileLocationDoc(fileLocation);
Response newDoc = db.save(doc);
if (newDoc.getStatusCode() == HttpStatus.SC_CREATED) {
return doc;
} else {
return null;
}
}
@Override
public FileListResponse findAll(FileListRequest request) {
int pageSize = request.getItems();
int pageNum = request.getPageNum();
log.info("request.getTimeFrom()...." + request.getTimeFrom());
log.info("request.getUserID()................" + request.getUserID());
log.info("request.getTimeTo()........" + request.getTimeTo());
QueryResult<FileLocationDoc> results = db.query(
new QueryBuilder(
and(gte("createdDate", toDate(request.getTimeFrom())),
lte("createdDate", toDate(request.getTimeTo())),
eq("createdBy", request.getUserID())))
.limit(pageSize).skip(pageSize * pageNum).build(),
FileLocationDoc.class);
if (results == null) {
throw new FileLocationNotFoundException(
format("Nothing found for such filter and page(num: %s, size: %s).", pageNum, pageSize));
}
List<FileLocation> content = results.getDocs().stream()
.map(FileLocationDoc::getFileLocation)
.collect(Collectors.toList());
return FileListResponse.builder()
.content(content)
.size(pageSize)
.number(pageNum)
.numberOfElements(content.size()).build();
}
private Long toDate(LocalDateTime dateTime) {
return Date.from(dateTime.toInstant(ZoneOffset.UTC)).getTime();
}
}
provider/file-ibm/src/main/java/org/opengroup/osdu/file/provider/ibm/security/IBMSecurityConfig.java 0 → 100644
View file @ aa666876
/*
* Copyright 2020 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.opengroup.osdu.file.provider.ibm.security;
import org.springframework.context.annotation.Configuration;
import org.springframework.security.config.annotation.method.configuration.EnableGlobalMethodSecurity;
import org.springframework.security.config.annotation.web.builders.HttpSecurity;
import org.springframework.security.config.annotation.web.configuration.EnableWebSecurity;
import org.springframework.security.config.annotation.web.configuration.WebSecurityConfigurerAdapter;
@Configuration
@EnableWebSecurity
@EnableGlobalMethodSecurity(prePostEnabled = true)
public class IBMSecurityConfig extends WebSecurityConfigurerAdapter {
@Override
protected void configure(HttpSecurity http) throws Exception {
http.httpBasic().disable()
.csrf().disable(); //disable default authN. AuthN handled by endpoints proxy
}
}
provider/file-ibm/src/main/java/org/opengroup/osdu/file/provider/ibm/service/ExpirationDateHelper.java 0 → 100644
View file @ aa666876
package org.opengroup.osdu.file.provider.ibm.service;
import java.util.Date;
// Needed to add this class to make it easy to unit test the creation of a date
// to the current time of running
public class ExpirationDateHelper {
public Date getExpirationDate(int s3SignedUrlExpirationTimeInDays){
Date expiration = new java.util.Date();
long expTimeMillis = expiration.getTime();
expTimeMillis += 1000 * 60 * 60 * 24 * s3SignedUrlExpirationTimeInDays;
expiration.setTime(expTimeMillis);
return expiration;
}
}
provider/file-ibm/src/main/java/org/opengroup/osdu/file/provider/ibm/service/IBMStorageServiceImpl.java 0 → 100644
View file @ aa666876
/*
* Copyright 2020 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.opengroup.osdu.file.provider.ibm.service;
import java.net.URI;
import java.net.URL;
import java.time.Instant;
import java.util.Date;
import java.util.HashMap;
import java.util.Map;
//import org.opengroup.osdu.file.provider.ibm.security.WhoamiController;
import javax.annotation.PostConstruct;
import javax.inject.Inject;
import org.apache.http.HttpStatus;
import org.opengroup.osdu.core.common.model.http.AppException;
import org.opengroup.osdu.core.common.model.http.DpsHeaders;
import org.opengroup.osdu.core.ibm.objectstorage.CloudObjectStorageFactory;
import org.opengroup.osdu.file.exception.OsduException;
import org.opengroup.osdu.file.model.SignedUrl;