IBM provider project created

provider/schema-ibm/docker/Dockerfile

+FROM openjdk:8-slim
+RUN apt-get update && apt-get install -y curl
+
+ENTRYPOINT ["java","-Djava.security.egd=file:/dev/./urandom","-jar","-Dspring.profiles.active=kuber","/app.jar"]
+EXPOSE 8080
+
+ARG JAR_FILE
+COPY ${JAR_FILE} app.jar
+

provider/schema-ibm/docker/docker-compose.yml

+version: "3"
+services:
+  os-schema-app:
+    build:
+      args: 
+        JAR_FILE: target/os-schema-gcp-0.0.1-spring-boot.jar
+      context: ..
+      dockerfile: docker/Dockerfile
+    image: gcr.io/opendes/os-schema-app
+    ports:
+     - "8080:8080"

provider/schema-ibm/pom.xml

+<project xmlns="http://maven.apache.org/POM/4.0.0"
+	xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+	xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
+	<modelVersion>4.0.0</modelVersion>
+	<parent>
+		<groupId>org.opengroup.osdu</groupId>
+		<artifactId>os-schema</artifactId>
+		<version>0.0.1</version>
+		<relativePath>../../</relativePath>
+	</parent>
+	<artifactId>os-schema-ibm</artifactId>
+	<description>IBM cloud related implementation staff.</description>
+	<packaging>jar</packaging>
+
+	<dependencies>
+		<!-- https://mvnrepository.com/artifact/com.google.cloud/google-cloud-storage -->
+	<!-- 	<dependency>
+			<groupId>com.google.cloud</groupId>
+			<artifactId>google-cloud-storage</artifactId>
+			<version>1.105.0</version>
+		</dependency>
+
+		https://mvnrepository.com/artifact/com.google.cloud/google-cloud-datastore
+		<dependency>
+			<groupId>com.google.cloud</groupId>
+			<artifactId>google-cloud-datastore</artifactId>
+			<version>1.102.2</version>
+		</dependency> -->
+
+		<dependency>
+			<groupId>org.opengroup.osdu</groupId>
+			<artifactId>os-schema-core</artifactId>
+			<version>0.0.1</version>
+		</dependency>
+
+		<dependency>
+			<groupId>org.springframework.boot</groupId>
+			<artifactId>spring-boot-starter-test</artifactId>
+			<scope>test</scope>
+		</dependency>
+
+		<dependency>
+			<groupId>org.mockito</groupId>
+			<artifactId>mockito-all</artifactId>
+			<version>1.10.19</version>
+			<scope>test</scope>
+		</dependency>
+
+		<!-- https://mvnrepository.com/artifact/com.google.apis/google-api-services-iam -->
+	<!-- 	<dependency>
+			<groupId>com.google.apis</groupId>
+			<artifactId>google-api-services-iam</artifactId>
+			<version>v1-rev310-1.25.0</version>
+		</dependency> -->
+
+		<!-- <dependency>
+			<groupId>org.opengroup.osdu</groupId>
+			<artifactId>core-lib-gcp</artifactId>
+			<version>0.1.15</version>
+		</dependency> -->
+		
+		<dependency>
+			<groupId>org.opengroup.osdu</groupId>
+			<artifactId>os-core-lib-ibm</artifactId>
+			<version>0.0.13-SNAPSHOT</version>
+		</dependency>
+
+		<dependency>
+			<groupId>io.cucumber</groupId>
+			<artifactId>cucumber-java8</artifactId>
+			<version>5.4.0</version>
+			<scope>test</scope>
+		</dependency>
+
+		<dependency>
+			<groupId>io.cucumber</groupId>
+			<artifactId>cucumber-junit</artifactId>
+			<version>5.4.0</version>
+			<scope>test</scope>
+		</dependency>
+
+		<dependency>
+			<groupId>io.cucumber</groupId>
+			<artifactId>cucumber-guice</artifactId>
+			<version>5.4.0</version>
+			<scope>test</scope>
+		</dependency> 
+
+		<dependency>
+			<groupId>com.google.inject</groupId>
+			<artifactId>guice</artifactId>
+			<version>4.2.0</version>
+			<scope>test</scope>
+		</dependency>
+
+	</dependencies>
+
+	<repositories>
+		<repository>
+			<id>${gitlab-server}</id>
+			<url>https://community.opengroup.org/api/v4/groups/17/-/packages/maven</url>
+		</repository>
+	</repositories>
+
+	<distributionManagement>
+		<repository>
+			<id>${gitlab-server}</id>
+			<url>https://community.opengroup.org/api/v4/projects/26/packages/maven</url>
+		</repository>
+		<snapshotRepository>
+			<id>${gitlab-server}</id>
+			<url>https://community.opengroup.org/api/v4/projects/26/packages/maven</url>
+		</snapshotRepository>
+	</distributionManagement>
+
+	<build>
+		<plugins>
+			<plugin>
+				<groupId>org.springframework.boot</groupId>
+				<artifactId>spring-boot-maven-plugin</artifactId>
+				<executions>
+					<execution>
+						<goals>
+							<goal>repackage</goal>
+						</goals>
+						<configuration>
+							<classifier>spring-boot</classifier>
+							<mainClass>
+								org.opengroup.osdu.schema.SchemaApplication
+							</mainClass>
+						</configuration>
+					</execution>
+
+				</executions>
+			</plugin>
+			<plugin>
+				<artifactId>maven-war-plugin</artifactId>
+				<configuration>
+					<failOnMissingWebXml>false</failOnMissingWebXml>
+				</configuration>
+			</plugin>
+
+			<plugin>
+				<groupId>org.apache.maven.plugins</groupId>
+				<artifactId>maven-surefire-plugin</artifactId>
+				<version>3.0.0-M3</version>
+				<configuration>
+					<skipTests>${skipUnitTests}</skipTests>
+					<forkCount>3</forkCount>
+					<reuseForks>true</reuseForks>
+					<argLine>-Xmx1024m -XX:MaxPermSize=256m</argLine>
+				</configuration>
+			</plugin>
+
+		</plugins>
+	</build>
+
+
+</project>
\ No newline at end of file

provider/schema-ibm/src/main/java/org/opengroup/osdu/schema/credentials/CloudCredentials.java

+package org.opengroup.osdu.schema.credentials;
+
+import java.io.IOException;
+import java.security.GeneralSecurityException;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.logging.Level;
+
+import org.apache.commons.lang3.StringUtils;
+import org.apache.commons.lang3.time.DateUtils;
+import org.apache.http.HttpHeaders;
+import org.opengroup.osdu.core.common.http.HttpClient;
+import org.opengroup.osdu.core.common.http.HttpRequest;
+import org.opengroup.osdu.core.common.http.HttpResponse;
+import org.opengroup.osdu.core.common.model.tenant.TenantInfo;
+import org.opengroup.osdu.schema.constants.SchemaConstants;
+import org.opengroup.osdu.schema.exceptions.ApplicationException;
+
+import com.google.api.client.googleapis.auth.oauth2.GoogleCredential;
+import com.google.api.client.googleapis.javanet.GoogleNetHttpTransport;
+import com.google.api.client.json.JsonFactory;
+import com.google.api.client.json.jackson2.JacksonFactory;
+import com.google.api.services.iam.v1.Iam;
+import com.google.api.services.iam.v1.Iam.Projects;
+import com.google.api.services.iam.v1.Iam.Projects.ServiceAccounts;
+import com.google.api.services.iam.v1.Iam.Projects.ServiceAccounts.SignJwt;
+import com.google.api.services.iam.v1.IamScopes;
+import com.google.api.services.iam.v1.model.SignJwtRequest;
+import com.google.api.services.iam.v1.model.SignJwtResponse;
+import com.google.auth.oauth2.AccessToken;
+import com.google.auth.oauth2.GoogleCredentials;
+import com.google.gson.JsonObject;
+
+import lombok.extern.java.Log;
+
+/**
+ * This class is used to implement the domain wide delegation using the service
+ * account token creator role. This class extends GoogleCredentials, and creates
+ * access token by creating the JWT token without private keys, and signing it
+ * with Google OAuth2 service.
+ * 
+ */
+
+@Log
+public class CloudCredentials extends GoogleCredentials {
+
+    private static final long serialVersionUID = -8461791038757192780L;
+
+    private static final String JWT_AUDIENCE = "https://www.googleapis.com/oauth2/v4/token";
+    private static final String SERVICE_ACCOUNT_NAME_FORMAT = "projects/%s/serviceAccounts/%s";
+    private static final String ACCESS_TOKEN = "access_token";
+    private static final String EXPIRES_IN = "expires_in";
+    private static final String SCOPE = "https://www.googleapis.com/auth/devstorage.full_control https://www.googleapis.com/auth/datastore";
+
+    private static final JsonFactory JSON_FACTORY = new JacksonFactory();
+
+    private final TenantInfo tenant;
+    private Iam iam;
+
+    public CloudCredentials(TenantInfo tenant) throws ApplicationException {
+
+        if ((tenant == null) || StringUtils.isBlank(tenant.getName()) || StringUtils.isBlank(tenant.getProjectId())
+                || StringUtils.isBlank(tenant.getServiceAccount())) {
+            throw new ApplicationException("Tenant name, project id or service account is not set.");
+        }
+        this.tenant = tenant;
+    }
+
+    /**
+     * Overrides the GoogleCredentials refreshAccessToken. Returns JWT token for the
+     * service account of the tenant.
+     * 
+     * @throws IOException
+     */
+    @Override
+    public AccessToken refreshAccessToken() throws IOException {
+
+        try {
+
+            Map<String, Object> signJwtPayload = this.getPayload();
+            String signedJwt = getSignedJwt(signJwtPayload);
+            return getAccessToken(signedJwt);
+
+        } catch (ApplicationException e) {
+            log.log(Level.SEVERE, "Error : An unexpected error occurred while geting refresh access token : ", e);
+            throw new IOException("An error occurred when accessing third-party APIs");
+        }
+
+    }
+
+    /**
+     * Creates the payload for JWT signing
+     * 
+     * @return payload as map
+     */
+    private Map<String, Object> getPayload() {
+
+        Map<String, Object> payload = new HashMap<>();
+
+        payload.put("scope", SCOPE);
+        // This will get current time in seconds and add 1 hours (i.e 3600 seconds to
+        // it)
+        payload.put("exp", System.currentTimeMillis() / 1000 + 3600);
+        payload.put("iat", System.currentTimeMillis() / 1000);
+        payload.put("iss", this.tenant.getServiceAccount());
+        payload.put("aud", JWT_AUDIENCE);
+
+        return payload;
+    }
+
+    /**
+     * Signs the JWT token of the tenant service account, without private keys.
+     * 
+     * @param signJwtPayload
+     * @return the signed JWT
+     * @throws ApplicationException
+     */
+    private String getSignedJwt(Map<String, Object> signJwtPayload) throws ApplicationException {
+        try {
+            SignJwtRequest signJwtRequest = new SignJwtRequest();
+            signJwtRequest.setPayload(JSON_FACTORY.toString(signJwtPayload));
+
+            String serviceAccountName = String.format(SERVICE_ACCOUNT_NAME_FORMAT, this.tenant.getProjectId(),
+                    this.tenant.getServiceAccount());
+
+            Iam iamInstance = this.getIam();
+            Projects projects = iamInstance.projects();
+            ServiceAccounts serviceAccounts = projects.serviceAccounts();
+            SignJwt signJwt = serviceAccounts.signJwt(serviceAccountName, signJwtRequest);
+
+            SignJwtResponse signJwtResponse = signJwt.execute();
+            return signJwtResponse.getSignedJwt();
+
+        } catch (IOException | GeneralSecurityException e) {
+            log.log(Level.SEVERE, "Error occoured: ", e);
+            throw new ApplicationException("An error occurred when accessing third-party APIs");
+        }
+
+    }
+
+    /**
+     * Creates the access token from the signed JWT for the tenant service account
+     * by signing it with Google OAuth2 service.
+     * 
+     * @param signedJwt the signed JWT
+     * @return Access token that is used by Google services such as GCS, PubSub,
+     *         Datastore at the time of the access
+     * @throws GeneralSecurityException
+     * @throws ApplicationException
+     */
+    private AccessToken getAccessToken(String signedJwt) throws ApplicationException {
+
+        HttpRequest request = HttpRequest.post().url(JWT_AUDIENCE)
+                .headers(Collections.singletonMap(HttpHeaders.CONTENT_TYPE, "application/x-www-form-urlencoded"))
+                .body(String.format("%s=%s&%s=%s", "grant_type", "urn:ietf:params:oauth:grant-type:jwt-bearer",
+                        "assertion", signedJwt))
+                .build();
+
+        HttpResponse response = new HttpClient().send(request);
+        JsonObject jsonResult = response.getAsJsonObject();
+
+        if (!jsonResult.has(ACCESS_TOKEN)) {
+            throw new ApplicationException("An error occurred when accessing third-party APIs");
+        }
+
+        return new AccessToken(jsonResult.get(ACCESS_TOKEN).getAsString(),
+                DateUtils.addSeconds(new Date(), jsonResult.get(EXPIRES_IN).getAsInt()));
+
+    }
+
+    /**
+     * Gets the Iam object of the services project, which is further used for
+     * signing the JWT
+     * 
+     * @return the Iam object of the services project
+     * @throws IOException
+     * @throws GeneralSecurityException
+     */
+    private Iam getIam() throws GeneralSecurityException, IOException {
+        if (this.iam == null) {
+
+            Iam.Builder builder = new Iam.Builder(GoogleNetHttpTransport.newTrustedTransport(), JSON_FACTORY,
+                    GoogleCredential.getApplicationDefault().createScoped(Arrays.asList(IamScopes.CLOUD_PLATFORM)))
+                            .setApplicationName(SchemaConstants.APPLICATION_NAME);
+
+            this.iam = builder.build();
+        }
+        return this.iam;
+    }
+}
\ No newline at end of file

provider/schema-ibm/src/main/java/org/opengroup/osdu/schema/credentials/DatastoreFactory.java

+package org.opengroup.osdu.schema.credentials;
+
+import java.util.concurrent.TimeUnit;
+
+import org.opengroup.osdu.core.common.model.tenant.TenantInfo;
+import org.opengroup.osdu.core.common.util.Crc32c;
+import org.opengroup.osdu.schema.exceptions.ApplicationException;
+import org.springframework.stereotype.Component;
+import org.threeten.bp.Duration;
+
+import com.google.api.gax.retrying.RetrySettings;
+import com.google.cloud.datastore.Datastore;
+import com.google.cloud.datastore.DatastoreOptions;
+import com.google.common.cache.Cache;
+import com.google.common.cache.CacheBuilder;
+
+@Component
+public class DatastoreFactory {
+
+    private Cache<String, CloudCredentials> cache = CacheBuilder.newBuilder()
+            .expireAfterAccess(60 * 60, TimeUnit.SECONDS).maximumSize(100).build();
+
+    // The numbers used for settings are selected on random basis, We can update
+    // them with experience/issue faced
+    private static final RetrySettings RETRY_SETTINGS = RetrySettings.newBuilder().setMaxAttempts(6)
+            .setInitialRetryDelay(Duration.ofSeconds(1)).setMaxRetryDelay(Duration.ofSeconds(10))
+            .setRetryDelayMultiplier(2.0).setTotalTimeout(Duration.ofSeconds(50))
+            .setInitialRpcTimeout(Duration.ofSeconds(50)).setRpcTimeoutMultiplier(1.1)
+            .setMaxRpcTimeout(Duration.ofSeconds(50)).build();
+
+    public Datastore getDatastore(TenantInfo tenantInfo) throws ApplicationException {
+
+        String cacheKey = this.getCredentialsCacheKey(tenantInfo.getName());
+        CloudCredentials credential = this.cache.getIfPresent(cacheKey);
+
+        if (credential == null) {
+            credential = new CloudCredentials(tenantInfo);
+            this.cache.put(cacheKey, credential);
+        }
+
+        return DatastoreOptions.newBuilder().setRetrySettings(RETRY_SETTINGS).setCredentials(credential)
+                .setProjectId(tenantInfo.getProjectId()).build().getService();
+    }
+
+    private String getCredentialsCacheKey(String tenantName) {
+        return Crc32c.hashToBase64EncodedString(String.format("CloudCredential:%s", tenantName));
+    }
+
+}
\ No newline at end of file

provider/schema-ibm/src/main/java/org/opengroup/osdu/schema/credentials/StorageFactory.java

+package org.opengroup.osdu.schema.credentials;
+
+import java.util.concurrent.TimeUnit;
+
+import org.opengroup.osdu.core.common.model.tenant.TenantInfo;
+import org.opengroup.osdu.schema.exceptions.ApplicationException;
+import org.springframework.stereotype.Component;
+import org.threeten.bp.Duration;
+
+import com.google.api.gax.retrying.RetrySettings;
+import com.google.cloud.TransportOptions;
+import com.google.cloud.http.HttpTransportOptions;
+import com.google.cloud.storage.Storage;
+import com.google.cloud.storage.StorageOptions;
+import com.google.common.cache.Cache;
+import com.google.common.cache.CacheBuilder;
+
+@Component
+public class StorageFactory {
+
+    // The numbers used for settings are selected on random basis, We can update
+    // them with experience/issue faced
+    private static final RetrySettings RETRY_SETTINGS = RetrySettings.newBuilder().setMaxAttempts(6)
+            .setInitialRetryDelay(Duration.ofSeconds(1)).setMaxRetryDelay(Duration.ofSeconds(10))
+            .setRetryDelayMultiplier(2.0).setTotalTimeout(Duration.ofSeconds(50))
+            .setInitialRpcTimeout(Duration.ofSeconds(50)).setRpcTimeoutMultiplier(1.1)
+            .setMaxRpcTimeout(Duration.ofSeconds(50)).build();
+
+    // The numbers used for settings are selected on random basis, We can update
+    // them with experience/issue faced
+    private static final TransportOptions TRANSPORT_OPTIONS = HttpTransportOptions.newBuilder()
+            .setReadTimeout(40 * 1000).setConnectTimeout(10 * 1000).build();
+
+    private Cache<String, CloudCredentials> cache = CacheBuilder.newBuilder()
+            .expireAfterAccess(60 * 60, TimeUnit.SECONDS).maximumSize(100).build();
+
+    public Storage getStorage(TenantInfo tenantInfo) throws ApplicationException {
+        CloudCredentials credential = this.cache.getIfPresent(tenantInfo.getName());
+        if (credential == null) {
+            credential = new CloudCredentials(tenantInfo);
+            this.cache.put(tenantInfo.getName(), credential);
+        }
+
+        return StorageOptions.newBuilder().setCredentials(credential).setProjectId(tenantInfo.getProjectId())
+                .setRetrySettings(RETRY_SETTINGS).setTransportOptions(TRANSPORT_OPTIONS).build().getService();
+    }
+}
\ No newline at end of file

provider/schema-ibm/src/main/java/org/opengroup/osdu/schema/impl/schemainfostore/IbmAuthorityStore.java

+package org.opengroup.osdu.schema.impl.schemainfostore;
+
+import java.text.MessageFormat;
+
+import org.opengroup.osdu.core.common.model.http.DpsHeaders;
+import org.opengroup.osdu.core.common.provider.interfaces.ITenantFactory;
+import org.opengroup.osdu.schema.constants.SchemaConstants;
+import org.opengroup.osdu.schema.credentials.DatastoreFactory;
+import org.opengroup.osdu.schema.exceptions.ApplicationException;
+import org.opengroup.osdu.schema.exceptions.BadRequestException;
+import org.opengroup.osdu.schema.exceptions.NotFoundException;
+import org.opengroup.osdu.schema.model.Authority;
+import org.opengroup.osdu.schema.provider.interfaces.schemainfostore.IAuthorityStore;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Repository;
+
+import com.google.cloud.Timestamp;
+import com.google.cloud.datastore.Datastore;
+import com.google.cloud.datastore.DatastoreException;
+import com.google.cloud.datastore.Entity;
+import com.google.cloud.datastore.Key;
+
+import lombok.extern.java.Log;
+
+/**
+ * Repository class to to register authority in Google store.
+ *
+ */
+
+@Log
+@Repository
+public class IbmAuthorityStore implements IAuthorityStore {
+
+    @Autowired
+    private DpsHeaders headers;
+
+    @Autowired
+    private DatastoreFactory dataStoreFactory;
+
+    @Autowired
+    private ITenantFactory tenantFactory;
+
+    /**
+     * Method to get authority from google store
+     *
+     * @param authorityId
+     * @return Authority object
+     * @throws ApplicationException
+     * @throws NotFoundException
+     */
+    @Override
+    public Authority get(String authorityId) throws NotFoundException, ApplicationException {
+        Datastore datastore = dataStoreFactory.getDatastore(tenantFactory.getTenantInfo(headers.getPartitionId()));
+        Key key = datastore.newKeyFactory().setNamespace(SchemaConstants.NAMESPACE)
+                .setKind(SchemaConstants.AUTHORITY_KIND).newKey(authorityId);
+
+        Entity entity = datastore.get(key);
+        if (entity == null) {
+            throw new NotFoundException("bad input parameter");
+        } else {
+            return mapEntityToDto(entity);
+        }
+
+    }
+
+    /**
+     * Method to create authority in google store of dataPartitionId project
+     *
+     * @param authority
+     * @param dataPartitionId
+     * @return Authority object
+     * @throws ApplicationException
+     * @throws BadRequestException
+     */
+    @Override
+    public Authority create(Authority authority) throws ApplicationException, BadRequestException {
+        Datastore datastore = dataStoreFactory.getDatastore(tenantFactory.getTenantInfo(headers.getPartitionId()));
+        Key key = datastore.newKeyFactory().setNamespace(SchemaConstants.NAMESPACE)
+                .setKind(SchemaConstants.AUTHORITY_KIND).newKey(authority.getAuthorityId());
+        Entity entity = getEntityObject(authority, key);
+        try {
+            datastore.add(entity);
+        } catch (DatastoreException ex) {
+            if ("ALREADY_EXISTS".equals(ex.getReason())) {
+                log.warning(SchemaConstants.AUTHORITY_EXISTS_ALREADY_REGISTERED);
+                throw new BadRequestException(
+                        MessageFormat.format(SchemaConstants.AUTHORITY_EXISTS_EXCEPTION, authority.getAuthorityId()));
+            } else {
+                log.severe(MessageFormat.format(SchemaConstants.OBJECT_INVALID, ex.getMessage()));
+                throw new ApplicationException(SchemaConstants.INVALID_INPUT);
+            }
+        }
+        log.info(SchemaConstants.AUTHORITY_CREATED);
+        return mapEntityToDto(entity);
+    }
+
+    private Authority mapEntityToDto(Entity entity) {
+
+        Auth