Skip to content
GitLab
Menu
Projects
Groups
Snippets
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Sign in / Register
Toggle navigation
Menu
Open sidebar
Open Subsurface Data Universe Software
Platform
Data Flow
Data Enrichment
wks
Commits
07db346b
Commit
07db346b
authored
Jan 26, 2021
by
harshit aggarwal
Browse files
making config changes
parent
19718cd7
Pipeline
#23571
failed with stages
in 2 minutes and 2 seconds
Changes
23
Pipelines
2
Hide whitespace changes
Inline
Side-by-side
devops/azure/chart/helm-config.yaml
View file @
07db346b
...
...
@@ -12,11 +12,10 @@ azure:
servicebusSubscription
:
eg_sb_wkssubscription
servicebusTopic
:
recordstopiceg
storageContainer
:
osdu-wks-mappings
queueLength
:
5
defaultTenant
:
#{DEFAULT_TENANT}#
image
:
branch
:
#{ENVIRONMENT_NAME}#
tag
:
#{Build.SourceVersion}#
repository
:
#{container-registry}#.azurecr.io
default_tenant
:
#{DEFAULT_TENANT}#
devops/azure/chart/templates/deployment.yaml
View file @
07db346b
...
...
@@ -100,10 +100,10 @@ spec:
-
name
:
partition_service_endpoint
value
:
"
http://partition/api/partition/v1"
-
name
:
default_tenant
value
:
{{
.Values.default
_t
enant
}}
value
:
{{
.Values.
azure.
default
T
enant
}}
-
name
:
max_concurrent_calls
value
:
"
64"
-
name
:
executor_n_threads
value
:
"
64"
-
name
:
max_lock_renew_duration_seconds
value
:
"
60
00"
value
:
"
3
00"
\ No newline at end of file
devops/azure/chart/values.yaml
View file @
07db346b
...
...
@@ -15,14 +15,14 @@
global
:
# Service(s) Replica Count
replicaCount
:
10
azure
:
servicebusSubscription
:
eg_sb_wkssubscription
servicebusTopic
:
recordstopiceg
storageContainer
:
osdu-wks-mappings
queueLength
:
5
defaultTenant
:
opendes
image
:
branch
:
master
tag
:
latest
repository
:
community.opengroup.org:5555/osdu/platform/data-flow/enrichment/wks
default_tenant
:
opendes
\ No newline at end of file
devops/azure/development-pipeline.yml
View file @
07db346b
...
...
@@ -55,8 +55,8 @@ stages:
parameters
:
mavenGoal
:
'
package'
mavenPublishJUnitResults
:
true
serviceCoreMavenOptions
:
'
-
P
wks-core
--settings
.mvn/community-maven-ado.settings.xml'
mavenOptions
:
'
-
P
wks-azure
--settings
.mvn/community-maven-ado.settings.xml
-Dmaven.repo.local=$(MAVEN_CACHE_FOLDER)'
serviceCoreMavenOptions
:
'
-
pl
wks-core
--settings
.mvn/community-maven-ado.settings.xml
-Dmaven.repo.local=$(MAVEN_CACHE_FOLDER)
'
mavenOptions
:
'
-
pl
provider/
wks-azure
--settings
.mvn/community-maven-ado.settings.xml
-Dmaven.repo.local=$(MAVEN_CACHE_FOLDER)'
copyFileContents
:
|
pom.xml
provider/wks-azure/maven/settings.xml
...
...
@@ -78,7 +78,7 @@ stages:
chartPath
:
${{ variables.chartPath }}
valuesFile
:
${{ variables.valuesFile }}
testCoreMavenPomFile
:
'
testing/wks-test-core/pom.xml'
testCoreMavenOptions
:
'
--settings
$(System.DefaultWorkingDirectory)/drop/.mvn/community-maven-ado.settings.xml
-DskipTests
-DskipITs'
testCoreMavenOptions
:
'
--settings
$(System.DefaultWorkingDirectory)/drop/.mvn/community-maven-ado.settings.xml
-Dmaven.repo.local=$(MAVEN_CACHE_FOLDER)
-DskipTests
-DskipITs'
integrationTestMavenGoal
:
'
verify'
skipDeploy
:
${{ variables.SKIP_DEPLOY }}
skipTest
:
${{ variables.SKIP_TESTS }}
...
...
devops/azure/pipeline.yml
View file @
07db346b
...
...
@@ -55,8 +55,8 @@ stages:
parameters
:
mavenGoal
:
'
package'
mavenPublishJUnitResults
:
true
serviceCoreMavenOptions
:
'
-
P
wks-core
--settings
.mvn/community-maven-ado.settings.xml'
mavenOptions
:
'
-
P
wks-azure
--settings
.mvn/community-maven-ado.settings.xml
-Dmaven.repo.local=$(MAVEN_CACHE_FOLDER)'
serviceCoreMavenOptions
:
'
-
pl
wks-core
--settings
.mvn/community-maven-ado.settings.xml
-Dmaven.repo.local=$(MAVEN_CACHE_FOLDER)
'
mavenOptions
:
'
-
pl
provider/
wks-azure
--settings
.mvn/community-maven-ado.settings.xml
-Dmaven.repo.local=$(MAVEN_CACHE_FOLDER)'
copyFileContents
:
|
pom.xml
provider/wks-azure/maven/settings.xml
...
...
@@ -78,7 +78,7 @@ stages:
chartPath
:
${{ variables.chartPath }}
valuesFile
:
${{ variables.valuesFile }}
testCoreMavenPomFile
:
'
testing/wks-test-core/pom.xml'
testCoreMavenOptions
:
'
--settings
$(System.DefaultWorkingDirectory)/drop/.mvn/community-maven-ado.settings.xml
-DskipTests
-DskipITs'
testCoreMavenOptions
:
'
--settings
$(System.DefaultWorkingDirectory)/drop/.mvn/community-maven-ado.settings.xml
-Dmaven.repo.local=$(MAVEN_CACHE_FOLDER)
-DskipTests
-DskipITs'
integrationTestMavenGoal
:
'
verify'
skipDeploy
:
${{ variables.SKIP_DEPLOY }}
skipTest
:
${{ variables.SKIP_TESTS }}
...
...
devops/azure/release.yaml
View file @
07db346b
File suppressed by a .gitattributes entry or the file's encoding is unsupported.
provider/wks-azure/src/main/java/org/opengroup/osdu/wks/provider/azure/WksServiceApplication.java
0 → 100644
View file @
07db346b
package
org.opengroup.osdu.wks.provider.azure
;
import
org.opengroup.osdu.wks.config.ThreadScopeBeanFactoryPostProcessor
;
import
org.opengroup.osdu.wks.provider.interfaces.SubscriptionManager
;
import
org.opengroup.osdu.wks.service.SchemaService
;
import
org.slf4j.Logger
;
import
org.slf4j.LoggerFactory
;
import
org.springframework.beans.factory.config.BeanFactoryPostProcessor
;
import
org.springframework.boot.SpringApplication
;
import
org.springframework.boot.autoconfigure.SpringBootApplication
;
import
org.springframework.context.ApplicationContext
;
import
org.springframework.context.annotation.Bean
;
import
org.springframework.context.annotation.ComponentScan
;
import
org.springframework.web.client.RestTemplate
;
@SpringBootApplication
@ComponentScan
(
value
=
{
"org.opengroup.osdu"
})
public
class
WksServiceApplication
{
private
final
static
Logger
LOGGER
=
LoggerFactory
.
getLogger
(
SchemaService
.
class
);
public
static
void
main
(
String
[]
args
)
{
ApplicationContext
context
=
SpringApplication
.
run
(
org
.
opengroup
.
osdu
.
wks
.
WksServiceApplication
.
class
,
args
);
try
{
SubscriptionManager
subscriptionManager
=
context
.
getBean
(
SubscriptionManager
.
class
);
subscriptionManager
.
subscribeRecordsChangeEvent
();
}
catch
(
Exception
e
)
{
LOGGER
.
error
(
e
.
getMessage
(),
e
);
}
}
@Bean
(
"restTemplate"
)
public
RestTemplate
restTemplate
()
{
return
new
RestTemplate
();
}
@Bean
public
static
BeanFactoryPostProcessor
beanFactoryPostProcessor
()
{
return
new
ThreadScopeBeanFactoryPostProcessor
();
}
}
provider/wks-azure/src/main/java/org/opengroup/osdu/wks/provider/azure/pubsub/SubscriptionManagerImpl.java
View file @
07db346b
...
...
@@ -60,7 +60,7 @@ public class SubscriptionManagerImpl implements SubscriptionManager {
new
MessageHandlerOptions
(
Integer
.
parseUnsignedInt
(
azureBootstrapConfig
.
getMaxConcurrentCalls
()),
false
,
Duration
.
ofSeconds
(
Integer
.
parseUnsignedInt
(
azureBootstrapConfig
.
getMaxLockRenewDurationInSeconds
())),
Duration
.
of
Millis
(
200
)
Duration
.
of
Seconds
(
1
)
),
executorService
);
...
...
provider/wks-azure/src/main/resources/application.properties
View file @
07db346b
...
...
@@ -53,13 +53,19 @@ spring.application.name=wks-azure
shared_tenant
=
${default_tenant}
# Storage service
STORAGE_API
=
${storage_service_endpoint}
osdu.host.storageServiceUrl
=
${storage_service_endpoint}
osdu.restclient.storageRetryIntervalInSeconds
=
1
osdu.restclient.storageRetryCounter
=
3
# Search service
SEARCH_API
=
${search_service_endpoint}
osdu.host.searchServiceUrl
=
${search_service_endpoint}
osdu.restclient.searchRetryIntervalInSeconds
=
1
osdu.restclient.searchRetryCounter
=
3
#Schema service
osdu.host.schemaServiceUrl
=
${schema_service_endpoint}
osdu.restclient.schemaRetryIntervalInSeconds
=
1
osdu.restclient.schemaRetryCounter
=
3
# Partition service
PARTITION_API
=
${partition_service_endpoint}
...
...
wks-core/src/main/java/org/opengroup/osdu/wks/constants/Constants.java
View file @
07db346b
...
...
@@ -24,6 +24,7 @@ public class Constants {
public
static
final
String
LAST_MODIFIED_DATE
=
"last_modified_date"
;
public
static
final
String
DASH
=
"-"
;
public
static
final
String
POINT
=
"."
;
public
static
final
String
Slash
=
"/"
;
public
static
final
String
MISSING_RAW_RECORD_DETAILS
=
"Missing raw records details"
;
public
static
final
String
DATA_PARTITION_ID_NEEDED
=
"Data Partition Id is needed"
;
public
static
final
String
CORRELATION_ID_NEEDED
=
"Correlation Id is needed"
;
...
...
wks-core/src/main/java/org/opengroup/osdu/wks/constants/SearchConstants.java
0 → 100644
View file @
07db346b
package
org.opengroup.osdu.wks.constants
;
public
class
SearchConstants
{
public
static
final
String
POST_SEARCH_ENDPOINT
=
"/query"
;
}
wks-core/src/main/java/org/opengroup/osdu/wks/constants/StorageConstants.java
0 → 100644
View file @
07db346b
package
org.opengroup.osdu.wks.constants
;
public
class
StorageConstants
{
public
static
final
String
PUT_STORAGE_ENDPOINT
=
"/records"
;
public
static
final
String
GET_STORAGE_ENDPOINT
=
"/records"
;
}
wks-core/src/main/java/org/opengroup/osdu/wks/service/RestClientConfig.java
0 → 100644
View file @
07db346b
package
org.opengroup.osdu.wks.service
;
import
lombok.Getter
;
import
lombok.Setter
;
import
org.springframework.beans.factory.annotation.Value
;
import
org.springframework.boot.context.properties.ConfigurationProperties
;
import
org.springframework.context.annotation.Configuration
;
@Configuration
@ConfigurationProperties
(
"osdu.restclient"
)
@Getter
@Setter
public
class
RestClientConfig
{
@Value
(
"#{new Integer('${searchRetryIntervalInSeconds:60}')}"
)
private
Integer
searchRetryIntervalInSeconds
;
@Value
(
"#{new Integer('${storageRetryIntervalInSeconds:60}')}"
)
private
Integer
storageRetryIntervalInSeconds
;
@Value
(
"#{new Integer('${schemaRetryIntervalInSeconds:60}')}"
)
private
Integer
schemaRetryIntervalInSeconds
;
@Value
(
"#{new Integer('${searchRetryCounter:3}')}"
)
private
Integer
searchRetryCounter
;
@Value
(
"#{new Integer('${storageRetryCounter:3}')}"
)
private
Integer
storageRetryCounter
;
@Value
(
"#{new Integer('${schemaRetryCounter:3}')}"
)
private
Integer
schemaRetryCounter
;
}
wks-core/src/main/java/org/opengroup/osdu/wks/service/SchemaService.java
View file @
07db346b
...
...
@@ -22,6 +22,7 @@ import org.slf4j.LoggerFactory;
import
org.springframework.beans.factory.annotation.Autowired
;
import
org.springframework.http.HttpMethod
;
import
org.springframework.stereotype.Component
;
import
org.springframework.web.client.RestTemplate
;
import
java.net.URISyntaxException
;
import
java.util.ArrayList
;
...
...
@@ -33,23 +34,31 @@ public class SchemaService implements ISchemaService{
private
final
static
Logger
LOGGER
=
LoggerFactory
.
getLogger
(
SchemaService
.
class
);
private
static
final
String
SCHEMA_SERVICE_ERROR_MESSAGE
=
"Schema service status code %s and message %s"
;
@Autowired
private
RestClient
restClient
;
@Autowired
private
UserCredential
userCredential
;
@Autowired
private
ITenantFactory
tenantFactory
;
private
final
RestClient
restClient
;
private
final
ServiceHostConfiguration
serviceHostConfiguration
;
private
final
UserCredential
userCredential
;
private
final
ITenantFactory
tenantFactory
;
private
final
RequestIdentity
requestIdentity
;
private
final
KindUtil
kindUtil
;
@Autowired
private
RequestIdentity
requestIdentity
;
@Autowired
private
ServiceHostConfiguration
serviceConfig
;
@Autowired
private
KindUtil
kindUtil
;
public
SchemaService
(
RestClient
restClient
,
RestClientConfig
restClientConfig
,
ServiceHostConfiguration
serviceHostConfiguration
,
KindUtil
kindUtil
,
RequestIdentity
requestIdentity
,
ITenantFactory
tenantFactory
,
UserCredential
userCredential
)
{
this
.
restClient
=
restClient
;
this
.
restClient
.
setRetryIntervalInSeconds
(
restClientConfig
.
getSchemaRetryIntervalInSeconds
());
this
.
restClient
.
setRetryCounter
(
restClientConfig
.
getSchemaRetryCounter
());
this
.
serviceHostConfiguration
=
serviceHostConfiguration
;
this
.
kindUtil
=
kindUtil
;
this
.
requestIdentity
=
requestIdentity
;
this
.
tenantFactory
=
tenantFactory
;
this
.
userCredential
=
userCredential
;
}
public
List
<
String
>
getSchemas
(
String
kind
)
throws
ApplicationException
{
RequestParameters
requestParameters
=
new
RequestParameters
();
...
...
@@ -106,7 +115,7 @@ public class SchemaService implements ISchemaService{
try
{
String
[]
kindArr
=
kind
.
split
(
Constants
.
COLON_SEPARATOR
);
String
majorVersion
=
kindUtil
.
getMajorVersionFromKind
(
kind
).
toString
();
String
schemaUrl
=
serviceConfig
.
getSchemaServiceUrl
()
+
SchemaConstants
.
POST_SCHEMA_ENDPOINT
;
String
schemaUrl
=
service
Host
Config
uration
.
getSchemaServiceUrl
()
.
concat
(
SchemaConstants
.
POST_SCHEMA_ENDPOINT
)
;
URIBuilder
uriBuilder
=
new
URIBuilder
(
schemaUrl
);
uriBuilder
.
addParameter
(
"authority"
,
kindArr
[
0
]);
uriBuilder
.
addParameter
(
"source"
,
kindArr
[
1
]);
...
...
wks-core/src/main/java/org/opengroup/osdu/wks/service/SearchService.java
View file @
07db346b
...
...
@@ -4,13 +4,13 @@ import org.apache.http.HttpStatus;
import
org.opengroup.osdu.core.common.model.tenant.TenantInfo
;
import
org.opengroup.osdu.core.common.provider.interfaces.ITenantFactory
;
import
org.opengroup.osdu.wks.config.RequestIdentity
;
import
org.opengroup.osdu.wks.constants.SearchConstants
;
import
org.opengroup.osdu.wks.exceptions.ApplicationException
;
import
org.opengroup.osdu.wks.model.RequestParameters
;
import
org.opengroup.osdu.wks.model.RestResponse
;
import
org.opengroup.osdu.wks.provider.interfaces.UserCredential
;
import
org.opengroup.osdu.wks.util.RestClient
;
import
org.springframework.beans.factory.annotation.Autowired
;
import
org.springframework.beans.factory.annotation.Value
;
import
org.springframework.http.HttpMethod
;
import
org.springframework.stereotype.Service
;
...
...
@@ -19,30 +19,33 @@ public class SearchService {
public
static
final
String
SEARCH_SERVICE_ERROR_MESSAGE
=
"Search service status code %s and message %s"
;
@Value
(
"${SEARCH_API}/query"
)
private
String
searchServiceUrl
;
private
final
RestClient
restClient
;
private
final
UserCredential
userCredential
;
private
final
ITenantFactory
tenantStorageFactory
;
private
final
RequestIdentity
requestIdentity
;
private
final
ServiceHostConfiguration
serviceHostConfiguration
;
@Autowired
public
SearchService
(
RestClient
restClient
,
UserCredential
userCredential
,
ITenantFactory
tenantStorageFactory
,
RequestIdentity
requestIdentity
)
{
this
.
restClient
=
restClient
;
this
.
userCredential
=
userCredential
;
this
.
tenantStorageFactory
=
tenantStorageFactory
;
this
.
requestIdentity
=
requestIdentity
;
}
@Autowired
public
SearchService
(
RestClient
restClient
,
RestClientConfig
restClientConfig
,
ServiceHostConfiguration
serviceHostConfiguration
,
UserCredential
userCredential
,
ITenantFactory
tenantStorageFactory
,
RequestIdentity
requestIdentity
)
{
this
.
restClient
=
restClient
;
this
.
restClient
.
setRetryIntervalInSeconds
(
restClientConfig
.
getSearchRetryIntervalInSeconds
());
this
.
restClient
.
setRetryCounter
(
restClientConfig
.
getSearchRetryCounter
());
this
.
serviceHostConfiguration
=
serviceHostConfiguration
;
this
.
userCredential
=
userCredential
;
this
.
tenantStorageFactory
=
tenantStorageFactory
;
this
.
requestIdentity
=
requestIdentity
;
}
public
String
query
(
String
payload
)
throws
ApplicationException
{
RequestParameters
requestParameters
=
new
RequestParameters
();
TenantInfo
tenantInfo
=
tenantStorageFactory
.
getTenantInfo
(
requestIdentity
.
getDataPartitionId
());
requestParameters
.
setUrl
(
se
archServiceUrl
);
requestParameters
.
setUrl
(
se
rviceHostConfiguration
.
getSearchServiceUrl
().
concat
(
SearchConstants
.
POST_SEARCH_ENDPOINT
)
);
requestParameters
.
setMethod
(
HttpMethod
.
POST
);
requestParameters
.
setAppKey
(
""
);
requestParameters
.
setAuthKey
(
userCredential
.
getIdToken
(
tenantInfo
));
...
...
wks-core/src/main/java/org/opengroup/osdu/wks/service/ServiceHostConfiguration.java
View file @
07db346b
...
...
@@ -10,9 +10,7 @@ import org.springframework.context.annotation.Configuration;
@Getter
@Setter
class
ServiceHostConfiguration
{
private
String
searchServiceUrl
;
private
String
storageServiceUrl
;
private
String
schemaServiceUrl
;
}
wks-core/src/main/java/org/opengroup/osdu/wks/service/StorageServiceImpl.java
View file @
07db346b
...
...
@@ -7,6 +7,8 @@ import org.apache.http.HttpStatus;
import
org.opengroup.osdu.core.common.model.tenant.TenantInfo
;
import
org.opengroup.osdu.core.common.provider.interfaces.ITenantFactory
;
import
org.opengroup.osdu.wks.config.RequestIdentity
;
import
org.opengroup.osdu.wks.constants.Constants
;
import
org.opengroup.osdu.wks.constants.StorageConstants
;
import
org.opengroup.osdu.wks.exceptions.ApplicationException
;
import
org.opengroup.osdu.wks.exceptions.AuthorizationException
;
import
org.opengroup.osdu.wks.model.RequestParameters
;
...
...
@@ -16,29 +18,37 @@ import org.opengroup.osdu.wks.util.RestClient;
import
org.slf4j.Logger
;
import
org.slf4j.LoggerFactory
;
import
org.springframework.beans.factory.annotation.Autowired
;
import
org.springframework.beans.factory.annotation.Value
;
import
org.springframework.http.HttpMethod
;
import
org.springframework.stereotype.Service
;
import
org.springframework.web.client.RestTemplate
;
@Service
public
class
StorageServiceImpl
implements
StorageService
{
private
final
static
Logger
LOGGER
=
LoggerFactory
.
getLogger
(
StorageServiceImpl
.
class
);
private
final
static
String
WKS_SUFFIX
=
"wks-source"
;
@Autowired
private
RestClient
restClient
;
@Autowired
private
UserCredential
userCren
denti
al
;
private
final
RestClient
restClient
;
private
final
ServiceHostConfiguration
serviceHostConfiguration
;
private
final
UserCredential
userCredential
;
private
final
ITenantFactory
tenantFactory
;
private
final
RequestIdentity
requestI
denti
ty
;
@Autowired
private
ITenantFactory
tenantFactory
;
@Autowired
private
RequestIdentity
requestIdentity
;
@Value
(
"${STORAGE_API}/records"
)
private
String
storageServiceUrl
;
public
StorageServiceImpl
(
RestClient
restClient
,
RestClientConfig
restClientConfig
,
ServiceHostConfiguration
serviceHostConfiguration
,
UserCredential
userCredential
,
RequestIdentity
requestIdentity
,
ITenantFactory
tenantFactory
)
{
this
.
restClient
=
restClient
;
this
.
restClient
.
setRetryIntervalInSeconds
(
restClientConfig
.
getStorageRetryIntervalInSeconds
());
this
.
restClient
.
setRetryCounter
(
restClientConfig
.
getStorageRetryCounter
());
this
.
serviceHostConfiguration
=
serviceHostConfiguration
;
this
.
userCredential
=
userCredential
;
this
.
requestIdentity
=
requestIdentity
;
this
.
tenantFactory
=
tenantFactory
;
}
/**
* This method returns the raw record for the given record id.
...
...
@@ -50,10 +60,11 @@ public class StorageServiceImpl implements StorageService {
@Override
public
Optional
<
String
>
getRecord
(
String
id
)
{
String
tenantName
=
requestIdentity
.
getDataPartitionId
();
String
getRecordUrl
=
serviceHostConfiguration
.
getStorageServiceUrl
().
concat
(
StorageConstants
.
GET_STORAGE_ENDPOINT
).
concat
(
Constants
.
Slash
).
concat
(
id
);
RequestParameters
requestParameters
=
new
RequestParameters
();
TenantInfo
tenantInfo
=
tenantFactory
.
getTenantInfo
(
tenantName
);
requestParameters
.
setUrl
(
storageServiceUrl
+
"/"
+
id
);
requestParameters
.
setUrl
(
getRecordUrl
);
requestParameters
.
setMethod
(
HttpMethod
.
GET
);
requestParameters
.
setAuthKey
(
getAuthKey
(
tenantInfo
));
requestParameters
.
setCorrelationId
(
requestIdentity
.
getCorrelationId
());
...
...
@@ -76,8 +87,10 @@ public class StorageServiceImpl implements StorageService {
@Override
public
String
putRecords
(
String
[]
jsonStrArr
,
String
tenantName
,
String
correlationId
)
throws
ApplicationException
{
RequestParameters
requestParameters
=
new
RequestParameters
();
String
putRecordUrl
=
serviceHostConfiguration
.
getStorageServiceUrl
().
concat
(
StorageConstants
.
PUT_STORAGE_ENDPOINT
);
TenantInfo
tenantInfo
=
tenantFactory
.
getTenantInfo
(
tenantName
);
requestParameters
.
setUrl
(
storageService
Url
);
requestParameters
.
setUrl
(
putRecord
Url
);
requestParameters
.
setMethod
(
HttpMethod
.
PUT
);
requestParameters
.
setAuthKey
(
getAuthKey
(
tenantInfo
));
// For Storage service put operation payload should be list which is change into
...
...
@@ -97,7 +110,7 @@ public class StorageServiceImpl implements StorageService {
private
String
getAuthKey
(
TenantInfo
tenantInfo
)
{
try
{
return
userCre
n
dential
.
getIdToken
(
tenantInfo
);
return
userCredential
.
getIdToken
(
tenantInfo
);
}
catch
(
ApplicationException
e
)
{
LOGGER
.
warn
(
e
.
getErrorMsg
(),
e
);
throw
new
AuthorizationException
(
e
);
...
...
wks-core/src/main/java/org/opengroup/osdu/wks/service/WKSServiceImpl.java
View file @
07db346b
...
...
@@ -120,7 +120,7 @@ public class WKSServiceImpl implements WKSService {
private
void
transformRecords
(
RawRecordDetails
[]
rawRecordsDetails
,
Map
<
String
,
List
<
MappingsModel
>>
mappingsMap
,
String
dataPartitionId
,
String
correlationId
)
throws
ApplicationException
,
BadRequestException
{
String
dataPartitionId
,
String
correlationId
)
throws
ApplicationException
{
List
<
Tree
>
wksRecordTreeList
=
new
ArrayList
<>();
List
<
RelationshipStatus
>
relationshipStatusList
=
new
ArrayList
<>();
for
(
RawRecordDetails
rawRecord
:
rawRecordsDetails
)
{
...
...
@@ -149,18 +149,27 @@ public class WKSServiceImpl implements WKSService {
catch
(
ApplicationException
e
)
{
LOGGER
.
warn
(
Constants
.
TRANSFORMATION_FAILED_FOR_GIVEN_WKS_KIND
,
rawRecord
.
getId
(),
mappings
.
getTargetSchemaKind
(),
e
.
getErrorMsg
(),
e
);
}
catch
(
Exception
e
)
{
LOGGER
.
warn
(
Constants
.
TRANSFORMATION_FAILED_FOR_GIVEN_WKS_KIND
,
rawRecord
.
getId
(),
mappings
.
getTargetSchemaKind
(),
e
.
getMessage
(),
e
);
}
}
}
catch
(
ApplicationException
e
)
{
LOGGER
.
warn
(
Constants
.
TRANFORMATION_FAILED
,
rawRecord
.
getId
(),
e
.
getErrorMsg
(),
e
);
}
catch
(
Exception
e
)
{
LOGGER
.
info
(
Constants
.
TRANFORMATION_FAILED
,
rawRecord
.
getId
(),
e
.
getMessage
(),
e
);
}
}
if
(!
wksRecordTreeList
.
isEmpty
())
{
saveAllWksRecords
(
wksRecordTreeList
,
dataPartitionId
,
correlationId
);
LOGGER
.
info
(
"Successfully saved {} WKS records in storage"
,
wksRecordTreeList
.
size
());
if
(!
relationshipStatusList
.
isEmpty
())
{
statusStoreService
.
createOrUpdateStoreEntry
(
relationshipStatusList
);
LOGGER
.
info
(
"Successfully saved relationship status for records for which WKS were not found"
);
}
}
}
...
...
wks-core/src/main/java/org/opengroup/osdu/wks/util/RelationshipBlockHandler.java
View file @
07db346b
...
...
@@ -68,8 +68,7 @@ public class RelationshipBlockHandler {
List
<
String
>
ids
=
relationship
.
getIds
();
rawIds
.
addAll
(
ids
);
}
List
<
Relationship
>
wksRelationships
=
fetchWksRelationships
(
rawRelationships
);
List
<
Relationship
>
wksRelationships
=
fetchWksRelationships
(
rawRelationships
);
Iterator
<
Relationship
>
relationshipIterator
=
rawRelationships
.
iterator
();
Iterator
<
Relationship
>
wksRelationshipIterator
=
wksRelationships
.
iterator
();
Set
<
String
>
pendingEntitiesToBeAdded
=
new
HashSet
<>();
...
...
@@ -91,46 +90,46 @@ public class RelationshipBlockHandler {
return
buildUpdatedRelationshipBlock
(
rawRelationshipsBlock
,
pendingEntitiesToBeAdded
,
pendingIdsToBeAdded
);
}
private
UpdatedRelationshipBlock
buildUpdatedRelationshipBlock
(
RelationshipsBlock
rawRelationshipsBlock
,
Set
<
String
>
pendingEntitiesToBeAdded
,
List
<
String
>
pendingIdsToBeAdded
)
{
UpdatedRelationshipBlock
updatedRelationshipBlock
=
new
UpdatedRelationshipBlock
();
private
UpdatedRelationshipBlock
buildUpdatedRelationshipBlock
(
RelationshipsBlock
rawRelationshipsBlock
,
Set
<
String
>
pendingEntitiesToBeAdded
,
List
<
String
>
pendingIdsToBeAdded
)
{
UpdatedRelationshipBlock
updatedRelationshipBlock
=
new
UpdatedRelationshipBlock
();
updatedRelationshipBlock
.
setRelationshipTree
(
rawRelationshipsBlock
.
getRelationshipBlockAsTree
());
updatedRelationshipBlock
.
setPendingIds
(
pendingIdsToBeAdded
);
updatedRelationshipBlock
.
setPendingEnitites
(
pendingEntitiesToBeAdded
);
return
updatedRelationshipBlock
;
}
}
private
List
<
Relationship
>
fetchWksRelationships
(
List
<
Relationship
>
rawRelationships
)
{
List
<
Relationship
>
wksRelationships
=
new
ArrayList
<>();
private
List
<
Relationship
>
fetchWksRelationships
(
List
<
Relationship
>
rawRelationships
)
{
List
<
Relationship
>
wksRelationships
=
new
ArrayList
<>();
for
(
Relationship
rawRelationship
:
rawRelationships
)
{
Relationship
wksRelationship
=
null
;
try
{
wksRelationship
=
getRelationshipBlockWithWksIds
(
rawRelationship
);
}
catch
(
ApplicationException
e
)
{
LOGGER
.
info
(
e
.
getMessage
(),
e
);
}
if
(
wksRelationship
!=
null
)
{
wksRelationships
.
add
(
wksRelationship
);
}
Relationship
wksRelationship
=
null
;
try
{
wksRelationship
=
getRelationshipBlockWithWksIds
(
rawRelationship
);
}
catch
(
ApplicationException
e
)
{
LOGGER
.
info
(
e
.
getMessage
(),
e
);
}
if
(
wksRelationship
!=
null
)
{
wksRelationships
.
add
(
wksRelationship
);
}
}
return
wksRelationships
;
}
return
wksRelationships
;
}
private
Relationship
getRelationshipBlockWithWksIds
(
Relationship
rawRelationship
)
throws
ApplicationException
{
List
<
String
>
rawIds
=
rawRelationship
.
getIds
();
List
<
String
>
wksIds
=
getWksIds
(
rawIds
);
private
Relationship
getRelationshipBlockWithWksIds
(
Relationship
rawRelationship
)
throws
ApplicationException
{
List
<
String
>
rawIds
=
rawRelationship
.
getIds
();
List
<
String
>
wksIds
=
getWksIds
(
rawIds
);
rawRelationship
.
setIds
(
wksIds
);
return
rawRelationship
;
}
rawRelationship
.
setIds
(
wksIds
);
return
rawRelationship
;
}
private
List
<
String
>
getWksIds
(
List
<
String
>
rawIds
)
{
return
rawIds
.
stream
().
map
(
this
::
getWksId
).
collect
(
Collectors
.
toList
());
}