Skip to content
GitLab
Menu
Projects
Groups
Snippets
/
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Sign in / Register
Toggle navigation
Menu
Open sidebar
Open Subsurface Data Universe Software
Platform
System
Schema
Commits
315de3bf
Commit
315de3bf
authored
Jun 08, 2020
by
Bhushan Rade
Browse files
IBM provider project created
parent
6ee872d1
Changes
20
Expand all
Hide whitespace changes
Inline
Side-by-side
provider/schema-ibm/docker/Dockerfile
0 → 100644
View file @
315de3bf
FROM
openjdk:8-slim
RUN
apt-get update
&&
apt-get
install
-y
curl
ENTRYPOINT
["java","-Djava.security.egd=file:/dev/./urandom","-jar","-Dspring.profiles.active=kuber","/app.jar"]
EXPOSE
8080
ARG
JAR_FILE
COPY
${JAR_FILE} app.jar
provider/schema-ibm/docker/docker-compose.yml
0 → 100644
View file @
315de3bf
version
:
"
3"
services
:
os-schema-app
:
build
:
args
:
JAR_FILE
:
target/os-schema-gcp-0.0.1-spring-boot.jar
context
:
..
dockerfile
:
docker/Dockerfile
image
:
gcr.io/opendes/os-schema-app
ports
:
-
"
8080:8080"
provider/schema-ibm/pom.xml
0 → 100644
View file @
315de3bf
<project
xmlns=
"http://maven.apache.org/POM/4.0.0"
xmlns:xsi=
"http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation=
"http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd"
>
<modelVersion>
4.0.0
</modelVersion>
<parent>
<groupId>
org.opengroup.osdu
</groupId>
<artifactId>
os-schema
</artifactId>
<version>
0.0.1
</version>
<relativePath>
../../
</relativePath>
</parent>
<artifactId>
os-schema-ibm
</artifactId>
<description>
IBM cloud related implementation staff.
</description>
<packaging>
jar
</packaging>
<dependencies>
<!-- https://mvnrepository.com/artifact/com.google.cloud/google-cloud-storage -->
<!-- <dependency>
<groupId>com.google.cloud</groupId>
<artifactId>google-cloud-storage</artifactId>
<version>1.105.0</version>
</dependency>
https://mvnrepository.com/artifact/com.google.cloud/google-cloud-datastore
<dependency>
<groupId>com.google.cloud</groupId>
<artifactId>google-cloud-datastore</artifactId>
<version>1.102.2</version>
</dependency> -->
<dependency>
<groupId>
org.opengroup.osdu
</groupId>
<artifactId>
os-schema-core
</artifactId>
<version>
0.0.1
</version>
</dependency>
<dependency>
<groupId>
org.springframework.boot
</groupId>
<artifactId>
spring-boot-starter-test
</artifactId>
<scope>
test
</scope>
</dependency>
<dependency>
<groupId>
org.mockito
</groupId>
<artifactId>
mockito-all
</artifactId>
<version>
1.10.19
</version>
<scope>
test
</scope>
</dependency>
<!-- https://mvnrepository.com/artifact/com.google.apis/google-api-services-iam -->
<!-- <dependency>
<groupId>com.google.apis</groupId>
<artifactId>google-api-services-iam</artifactId>
<version>v1-rev310-1.25.0</version>
</dependency> -->
<!-- <dependency>
<groupId>org.opengroup.osdu</groupId>
<artifactId>core-lib-gcp</artifactId>
<version>0.1.15</version>
</dependency> -->
<dependency>
<groupId>
org.opengroup.osdu
</groupId>
<artifactId>
os-core-lib-ibm
</artifactId>
<version>
0.0.13-SNAPSHOT
</version>
</dependency>
<dependency>
<groupId>
io.cucumber
</groupId>
<artifactId>
cucumber-java8
</artifactId>
<version>
5.4.0
</version>
<scope>
test
</scope>
</dependency>
<dependency>
<groupId>
io.cucumber
</groupId>
<artifactId>
cucumber-junit
</artifactId>
<version>
5.4.0
</version>
<scope>
test
</scope>
</dependency>
<dependency>
<groupId>
io.cucumber
</groupId>
<artifactId>
cucumber-guice
</artifactId>
<version>
5.4.0
</version>
<scope>
test
</scope>
</dependency>
<dependency>
<groupId>
com.google.inject
</groupId>
<artifactId>
guice
</artifactId>
<version>
4.2.0
</version>
<scope>
test
</scope>
</dependency>
</dependencies>
<repositories>
<repository>
<id>
${gitlab-server}
</id>
<url>
https://community.opengroup.org/api/v4/groups/17/-/packages/maven
</url>
</repository>
</repositories>
<distributionManagement>
<repository>
<id>
${gitlab-server}
</id>
<url>
https://community.opengroup.org/api/v4/projects/26/packages/maven
</url>
</repository>
<snapshotRepository>
<id>
${gitlab-server}
</id>
<url>
https://community.opengroup.org/api/v4/projects/26/packages/maven
</url>
</snapshotRepository>
</distributionManagement>
<build>
<plugins>
<plugin>
<groupId>
org.springframework.boot
</groupId>
<artifactId>
spring-boot-maven-plugin
</artifactId>
<executions>
<execution>
<goals>
<goal>
repackage
</goal>
</goals>
<configuration>
<classifier>
spring-boot
</classifier>
<mainClass>
org.opengroup.osdu.schema.SchemaApplication
</mainClass>
</configuration>
</execution>
</executions>
</plugin>
<plugin>
<artifactId>
maven-war-plugin
</artifactId>
<configuration>
<failOnMissingWebXml>
false
</failOnMissingWebXml>
</configuration>
</plugin>
<plugin>
<groupId>
org.apache.maven.plugins
</groupId>
<artifactId>
maven-surefire-plugin
</artifactId>
<version>
3.0.0-M3
</version>
<configuration>
<skipTests>
${skipUnitTests}
</skipTests>
<forkCount>
3
</forkCount>
<reuseForks>
true
</reuseForks>
<argLine>
-Xmx1024m -XX:MaxPermSize=256m
</argLine>
</configuration>
</plugin>
</plugins>
</build>
</project>
\ No newline at end of file
provider/schema-ibm/src/main/java/org/opengroup/osdu/schema/credentials/CloudCredentials.java
0 → 100644
View file @
315de3bf
package
org.opengroup.osdu.schema.credentials
;
import
java.io.IOException
;
import
java.security.GeneralSecurityException
;
import
java.util.Arrays
;
import
java.util.Collections
;
import
java.util.Date
;
import
java.util.HashMap
;
import
java.util.Map
;
import
java.util.logging.Level
;
import
org.apache.commons.lang3.StringUtils
;
import
org.apache.commons.lang3.time.DateUtils
;
import
org.apache.http.HttpHeaders
;
import
org.opengroup.osdu.core.common.http.HttpClient
;
import
org.opengroup.osdu.core.common.http.HttpRequest
;
import
org.opengroup.osdu.core.common.http.HttpResponse
;
import
org.opengroup.osdu.core.common.model.tenant.TenantInfo
;
import
org.opengroup.osdu.schema.constants.SchemaConstants
;
import
org.opengroup.osdu.schema.exceptions.ApplicationException
;
import
com.google.api.client.googleapis.auth.oauth2.GoogleCredential
;
import
com.google.api.client.googleapis.javanet.GoogleNetHttpTransport
;
import
com.google.api.client.json.JsonFactory
;
import
com.google.api.client.json.jackson2.JacksonFactory
;
import
com.google.api.services.iam.v1.Iam
;
import
com.google.api.services.iam.v1.Iam.Projects
;
import
com.google.api.services.iam.v1.Iam.Projects.ServiceAccounts
;
import
com.google.api.services.iam.v1.Iam.Projects.ServiceAccounts.SignJwt
;
import
com.google.api.services.iam.v1.IamScopes
;
import
com.google.api.services.iam.v1.model.SignJwtRequest
;
import
com.google.api.services.iam.v1.model.SignJwtResponse
;
import
com.google.auth.oauth2.AccessToken
;
import
com.google.auth.oauth2.GoogleCredentials
;
import
com.google.gson.JsonObject
;
import
lombok.extern.java.Log
;
/**
* This class is used to implement the domain wide delegation using the service
* account token creator role. This class extends GoogleCredentials, and creates
* access token by creating the JWT token without private keys, and signing it
* with Google OAuth2 service.
*
*/
@Log
public
class
CloudCredentials
extends
GoogleCredentials
{
private
static
final
long
serialVersionUID
=
-
8461791038757192780L
;
private
static
final
String
JWT_AUDIENCE
=
"https://www.googleapis.com/oauth2/v4/token"
;
private
static
final
String
SERVICE_ACCOUNT_NAME_FORMAT
=
"projects/%s/serviceAccounts/%s"
;
private
static
final
String
ACCESS_TOKEN
=
"access_token"
;
private
static
final
String
EXPIRES_IN
=
"expires_in"
;
private
static
final
String
SCOPE
=
"https://www.googleapis.com/auth/devstorage.full_control https://www.googleapis.com/auth/datastore"
;
private
static
final
JsonFactory
JSON_FACTORY
=
new
JacksonFactory
();
private
final
TenantInfo
tenant
;
private
Iam
iam
;
public
CloudCredentials
(
TenantInfo
tenant
)
throws
ApplicationException
{
if
((
tenant
==
null
)
||
StringUtils
.
isBlank
(
tenant
.
getName
())
||
StringUtils
.
isBlank
(
tenant
.
getProjectId
())
||
StringUtils
.
isBlank
(
tenant
.
getServiceAccount
()))
{
throw
new
ApplicationException
(
"Tenant name, project id or service account is not set."
);
}
this
.
tenant
=
tenant
;
}
/**
* Overrides the GoogleCredentials refreshAccessToken. Returns JWT token for the
* service account of the tenant.
*
* @throws IOException
*/
@Override
public
AccessToken
refreshAccessToken
()
throws
IOException
{
try
{
Map
<
String
,
Object
>
signJwtPayload
=
this
.
getPayload
();
String
signedJwt
=
getSignedJwt
(
signJwtPayload
);
return
getAccessToken
(
signedJwt
);
}
catch
(
ApplicationException
e
)
{
log
.
log
(
Level
.
SEVERE
,
"Error : An unexpected error occurred while geting refresh access token : "
,
e
);
throw
new
IOException
(
"An error occurred when accessing third-party APIs"
);
}
}
/**
* Creates the payload for JWT signing
*
* @return payload as map
*/
private
Map
<
String
,
Object
>
getPayload
()
{
Map
<
String
,
Object
>
payload
=
new
HashMap
<>();
payload
.
put
(
"scope"
,
SCOPE
);
// This will get current time in seconds and add 1 hours (i.e 3600 seconds to
// it)
payload
.
put
(
"exp"
,
System
.
currentTimeMillis
()
/
1000
+
3600
);
payload
.
put
(
"iat"
,
System
.
currentTimeMillis
()
/
1000
);
payload
.
put
(
"iss"
,
this
.
tenant
.
getServiceAccount
());
payload
.
put
(
"aud"
,
JWT_AUDIENCE
);
return
payload
;
}
/**
* Signs the JWT token of the tenant service account, without private keys.
*
* @param signJwtPayload
* @return the signed JWT
* @throws ApplicationException
*/
private
String
getSignedJwt
(
Map
<
String
,
Object
>
signJwtPayload
)
throws
ApplicationException
{
try
{
SignJwtRequest
signJwtRequest
=
new
SignJwtRequest
();
signJwtRequest
.
setPayload
(
JSON_FACTORY
.
toString
(
signJwtPayload
));
String
serviceAccountName
=
String
.
format
(
SERVICE_ACCOUNT_NAME_FORMAT
,
this
.
tenant
.
getProjectId
(),
this
.
tenant
.
getServiceAccount
());
Iam
iamInstance
=
this
.
getIam
();
Projects
projects
=
iamInstance
.
projects
();
ServiceAccounts
serviceAccounts
=
projects
.
serviceAccounts
();
SignJwt
signJwt
=
serviceAccounts
.
signJwt
(
serviceAccountName
,
signJwtRequest
);
SignJwtResponse
signJwtResponse
=
signJwt
.
execute
();
return
signJwtResponse
.
getSignedJwt
();
}
catch
(
IOException
|
GeneralSecurityException
e
)
{
log
.
log
(
Level
.
SEVERE
,
"Error occoured: "
,
e
);
throw
new
ApplicationException
(
"An error occurred when accessing third-party APIs"
);
}
}
/**
* Creates the access token from the signed JWT for the tenant service account
* by signing it with Google OAuth2 service.
*
* @param signedJwt the signed JWT
* @return Access token that is used by Google services such as GCS, PubSub,
* Datastore at the time of the access
* @throws GeneralSecurityException
* @throws ApplicationException
*/
private
AccessToken
getAccessToken
(
String
signedJwt
)
throws
ApplicationException
{
HttpRequest
request
=
HttpRequest
.
post
().
url
(
JWT_AUDIENCE
)
.
headers
(
Collections
.
singletonMap
(
HttpHeaders
.
CONTENT_TYPE
,
"application/x-www-form-urlencoded"
))
.
body
(
String
.
format
(
"%s=%s&%s=%s"
,
"grant_type"
,
"urn:ietf:params:oauth:grant-type:jwt-bearer"
,
"assertion"
,
signedJwt
))
.
build
();
HttpResponse
response
=
new
HttpClient
().
send
(
request
);
JsonObject
jsonResult
=
response
.
getAsJsonObject
();
if
(!
jsonResult
.
has
(
ACCESS_TOKEN
))
{
throw
new
ApplicationException
(
"An error occurred when accessing third-party APIs"
);
}
return
new
AccessToken
(
jsonResult
.
get
(
ACCESS_TOKEN
).
getAsString
(),
DateUtils
.
addSeconds
(
new
Date
(),
jsonResult
.
get
(
EXPIRES_IN
).
getAsInt
()));
}
/**
* Gets the Iam object of the services project, which is further used for
* signing the JWT
*
* @return the Iam object of the services project
* @throws IOException
* @throws GeneralSecurityException
*/
private
Iam
getIam
()
throws
GeneralSecurityException
,
IOException
{
if
(
this
.
iam
==
null
)
{
Iam
.
Builder
builder
=
new
Iam
.
Builder
(
GoogleNetHttpTransport
.
newTrustedTransport
(),
JSON_FACTORY
,
GoogleCredential
.
getApplicationDefault
().
createScoped
(
Arrays
.
asList
(
IamScopes
.
CLOUD_PLATFORM
)))
.
setApplicationName
(
SchemaConstants
.
APPLICATION_NAME
);
this
.
iam
=
builder
.
build
();
}
return
this
.
iam
;
}
}
\ No newline at end of file
provider/schema-ibm/src/main/java/org/opengroup/osdu/schema/credentials/DatastoreFactory.java
0 → 100644
View file @
315de3bf
package
org.opengroup.osdu.schema.credentials
;
import
java.util.concurrent.TimeUnit
;
import
org.opengroup.osdu.core.common.model.tenant.TenantInfo
;
import
org.opengroup.osdu.core.common.util.Crc32c
;
import
org.opengroup.osdu.schema.exceptions.ApplicationException
;
import
org.springframework.stereotype.Component
;
import
org.threeten.bp.Duration
;
import
com.google.api.gax.retrying.RetrySettings
;
import
com.google.cloud.datastore.Datastore
;
import
com.google.cloud.datastore.DatastoreOptions
;
import
com.google.common.cache.Cache
;
import
com.google.common.cache.CacheBuilder
;
@Component
public
class
DatastoreFactory
{
private
Cache
<
String
,
CloudCredentials
>
cache
=
CacheBuilder
.
newBuilder
()
.
expireAfterAccess
(
60
*
60
,
TimeUnit
.
SECONDS
).
maximumSize
(
100
).
build
();
// The numbers used for settings are selected on random basis, We can update
// them with experience/issue faced
private
static
final
RetrySettings
RETRY_SETTINGS
=
RetrySettings
.
newBuilder
().
setMaxAttempts
(
6
)
.
setInitialRetryDelay
(
Duration
.
ofSeconds
(
1
)).
setMaxRetryDelay
(
Duration
.
ofSeconds
(
10
))
.
setRetryDelayMultiplier
(
2.0
).
setTotalTimeout
(
Duration
.
ofSeconds
(
50
))
.
setInitialRpcTimeout
(
Duration
.
ofSeconds
(
50
)).
setRpcTimeoutMultiplier
(
1.1
)
.
setMaxRpcTimeout
(
Duration
.
ofSeconds
(
50
)).
build
();
public
Datastore
getDatastore
(
TenantInfo
tenantInfo
)
throws
ApplicationException
{
String
cacheKey
=
this
.
getCredentialsCacheKey
(
tenantInfo
.
getName
());
CloudCredentials
credential
=
this
.
cache
.
getIfPresent
(
cacheKey
);
if
(
credential
==
null
)
{
credential
=
new
CloudCredentials
(
tenantInfo
);
this
.
cache
.
put
(
cacheKey
,
credential
);
}
return
DatastoreOptions
.
newBuilder
().
setRetrySettings
(
RETRY_SETTINGS
).
setCredentials
(
credential
)
.
setProjectId
(
tenantInfo
.
getProjectId
()).
build
().
getService
();
}
private
String
getCredentialsCacheKey
(
String
tenantName
)
{
return
Crc32c
.
hashToBase64EncodedString
(
String
.
format
(
"CloudCredential:%s"
,
tenantName
));
}
}
\ No newline at end of file
provider/schema-ibm/src/main/java/org/opengroup/osdu/schema/credentials/StorageFactory.java
0 → 100644
View file @
315de3bf
package
org.opengroup.osdu.schema.credentials
;
import
java.util.concurrent.TimeUnit
;
import
org.opengroup.osdu.core.common.model.tenant.TenantInfo
;
import
org.opengroup.osdu.schema.exceptions.ApplicationException
;
import
org.springframework.stereotype.Component
;
import
org.threeten.bp.Duration
;
import
com.google.api.gax.retrying.RetrySettings
;
import
com.google.cloud.TransportOptions
;
import
com.google.cloud.http.HttpTransportOptions
;
import
com.google.cloud.storage.Storage
;
import
com.google.cloud.storage.StorageOptions
;
import
com.google.common.cache.Cache
;
import
com.google.common.cache.CacheBuilder
;
@Component
public
class
StorageFactory
{
// The numbers used for settings are selected on random basis, We can update
// them with experience/issue faced
private
static
final
RetrySettings
RETRY_SETTINGS
=
RetrySettings
.
newBuilder
().
setMaxAttempts
(
6
)
.
setInitialRetryDelay
(
Duration
.
ofSeconds
(
1
)).
setMaxRetryDelay
(
Duration
.
ofSeconds
(
10
))
.
setRetryDelayMultiplier
(
2.0
).
setTotalTimeout
(
Duration
.
ofSeconds
(
50
))
.
setInitialRpcTimeout
(
Duration
.
ofSeconds
(
50
)).
setRpcTimeoutMultiplier
(
1.1
)
.
setMaxRpcTimeout
(
Duration
.
ofSeconds
(
50
)).
build
();
// The numbers used for settings are selected on random basis, We can update
// them with experience/issue faced
private
static
final
TransportOptions
TRANSPORT_OPTIONS
=
HttpTransportOptions
.
newBuilder
()
.
setReadTimeout
(
40
*
1000
).
setConnectTimeout
(
10
*
1000
).
build
();
private
Cache
<
String
,
CloudCredentials
>
cache
=
CacheBuilder
.
newBuilder
()
.
expireAfterAccess
(
60
*
60
,
TimeUnit
.
SECONDS
).
maximumSize
(
100
).
build
();
public
Storage
getStorage
(
TenantInfo
tenantInfo
)
throws
ApplicationException
{
CloudCredentials
credential
=
this
.
cache
.
getIfPresent
(
tenantInfo
.
getName
());
if
(
credential
==
null
)
{
credential
=
new
CloudCredentials
(
tenantInfo
);
this
.
cache
.
put
(
tenantInfo
.
getName
(),
credential
);
}
return
StorageOptions
.
newBuilder
().
setCredentials
(
credential
).
setProjectId
(
tenantInfo
.
getProjectId
())
.
setRetrySettings
(
RETRY_SETTINGS
).
setTransportOptions
(
TRANSPORT_OPTIONS
).
build
().
getService
();
}
}
\ No newline at end of file
provider/schema-ibm/src/main/java/org/opengroup/osdu/schema/impl/schemainfostore/IbmAuthorityStore.java
0 → 100644
View file @
315de3bf
package
org.opengroup.osdu.schema.impl.schemainfostore
;
import
java.text.MessageFormat
;
import
org.opengroup.osdu.core.common.model.http.DpsHeaders
;
import
org.opengroup.osdu.core.common.provider.interfaces.ITenantFactory
;
import
org.opengroup.osdu.schema.constants.SchemaConstants
;
import
org.opengroup.osdu.schema.credentials.DatastoreFactory
;
import
org.opengroup.osdu.schema.exceptions.ApplicationException
;
import
org.opengroup.osdu.schema.exceptions.BadRequestException
;
import
org.opengroup.osdu.schema.exceptions.NotFoundException
;
import
org.opengroup.osdu.schema.model.Authority
;
import
org.opengroup.osdu.schema.provider.interfaces.schemainfostore.IAuthorityStore
;
import
org.springframework.beans.factory.annotation.Autowired
;
import
org.springframework.stereotype.Repository
;
import
com.google.cloud.Timestamp
;
import
com.google.cloud.datastore.Datastore
;
import
com.google.cloud.datastore.DatastoreException
;
import
com.google.cloud.datastore.Entity
;
import
com.google.cloud.datastore.Key
;
import
lombok.extern.java.Log
;
/**
* Repository class to to register authority in Google store.
*
*/
@Log
@Repository
public
class
IbmAuthorityStore
implements
IAuthorityStore
{
@Autowired
private
DpsHeaders
headers
;
@Autowired
private
DatastoreFactory
dataStoreFactory
;
@Autowired
private
ITenantFactory
tenantFactory
;
/**
* Method to get authority from google store
*
* @param authorityId
* @return Authority object
* @throws ApplicationException
* @throws NotFoundException
*/
@Override
public
Authority
get
(
String
authorityId
)
throws
NotFoundException
,
ApplicationException
{
Datastore
datastore
=
dataStoreFactory
.
getDatastore
(
tenantFactory
.
getTenantInfo
(
headers
.
getPartitionId
()));
Key
key
=
datastore
.
newKeyFactory
().
setNamespace
(
SchemaConstants
.
NAMESPACE
)
.
setKind
(
SchemaConstants
.
AUTHORITY_KIND
).
newKey
(
authorityId
);
Entity
entity
=
datastore
.
get
(
key
);
if
(
entity
==
null
)
{
throw
new
NotFoundException
(
"bad input parameter"
);
}
else
{
return
mapEntityToDto
(
entity
);
}
}
/**
* Method to create authority in google store of dataPartitionId project
*
* @param authority
* @param dataPartitionId
* @return Authority object
* @throws ApplicationException
* @throws BadRequestException
*/
@Override
public
Authority
create
(
Authority
authority
)
throws
ApplicationException
,
BadRequestException
{
Datastore
datastore
=
dataStoreFactory
.
getDatastore
(
tenantFactory
.
getTenantInfo
(
headers
.
getPartitionId
()));
Key
key
=
datastore
.
newKeyFactory
().
setNamespace
(
SchemaConstants
.
NAMESPACE
)
.
setKind
(
SchemaConstants
.
AUTHORITY_KIND
).
newKey
(
authority
.
getAuthorityId
());
Entity
entity
=
getEntityObject
(
authority
,
key
);
try
{
datastore
.
add
(
entity
);
}
catch
(
DatastoreException
ex
)
{
if
(
"ALREADY_EXISTS"
.
equals
(
ex
.
getReason
()))
{
log
.
warning
(
SchemaConstants
.
AUTHORITY_EXISTS_ALREADY_REGISTERED
);
throw
new
BadRequestException
(
MessageFormat
.
format
(
SchemaConstants
.
AUTHORITY_EXISTS_EXCEPTION
,
authority
.
getAuthorityId
()));
}
else
{
log
.
severe
(
MessageFormat
.
format
(
SchemaConstants
.
OBJECT_INVALID
,
ex
.
getMessage
()));
throw
new
ApplicationException
(
SchemaConstants
.
INVALID_INPUT
);
}
}
log
.
info
(
SchemaConstants
.
AUTHORITY_CREATED
);
return
mapEntityToDto
(
entity
);
}
private
Authority
mapEntityToDto
(
Entity
entity
)
{
Auth