Commit 07db346b authored by harshit aggarwal's avatar harshit aggarwal
Browse files

making config changes

parent 19718cd7
Pipeline #23571 failed with stages
in 2 minutes and 2 seconds
......@@ -12,11 +12,10 @@ azure:
servicebusSubscription: eg_sb_wkssubscription
servicebusTopic: recordstopiceg
storageContainer: osdu-wks-mappings
queueLength: 5
defaultTenant: #{DEFAULT_TENANT}#
image:
branch: #{ENVIRONMENT_NAME}#
tag: #{Build.SourceVersion}#
repository: #{container-registry}#.azurecr.io
default_tenant: #{DEFAULT_TENANT}#
......@@ -100,10 +100,10 @@ spec:
- name: partition_service_endpoint
value: "http://partition/api/partition/v1"
- name: default_tenant
value: {{ .Values.default_tenant }}
value: {{ .Values.azure.defaultTenant }}
- name: max_concurrent_calls
value: "64"
- name: executor_n_threads
value: "64"
- name: max_lock_renew_duration_seconds
value: "6000"
value: "300"
\ No newline at end of file
......@@ -15,14 +15,14 @@
global:
# Service(s) Replica Count
replicaCount: 10
azure:
servicebusSubscription: eg_sb_wkssubscription
servicebusTopic: recordstopiceg
storageContainer: osdu-wks-mappings
queueLength: 5
defaultTenant: opendes
image:
branch: master
tag: latest
repository: community.opengroup.org:5555/osdu/platform/data-flow/enrichment/wks
default_tenant: opendes
\ No newline at end of file
......@@ -55,8 +55,8 @@ stages:
parameters:
mavenGoal: 'package'
mavenPublishJUnitResults: true
serviceCoreMavenOptions: '-P wks-core --settings .mvn/community-maven-ado.settings.xml'
mavenOptions: '-P wks-azure --settings .mvn/community-maven-ado.settings.xml -Dmaven.repo.local=$(MAVEN_CACHE_FOLDER)'
serviceCoreMavenOptions: '-pl wks-core --settings .mvn/community-maven-ado.settings.xml -Dmaven.repo.local=$(MAVEN_CACHE_FOLDER)'
mavenOptions: '-pl provider/wks-azure --settings .mvn/community-maven-ado.settings.xml -Dmaven.repo.local=$(MAVEN_CACHE_FOLDER)'
copyFileContents: |
pom.xml
provider/wks-azure/maven/settings.xml
......@@ -78,7 +78,7 @@ stages:
chartPath: ${{ variables.chartPath }}
valuesFile: ${{ variables.valuesFile }}
testCoreMavenPomFile: 'testing/wks-test-core/pom.xml'
testCoreMavenOptions: '--settings $(System.DefaultWorkingDirectory)/drop/.mvn/community-maven-ado.settings.xml -DskipTests -DskipITs'
testCoreMavenOptions: '--settings $(System.DefaultWorkingDirectory)/drop/.mvn/community-maven-ado.settings.xml -Dmaven.repo.local=$(MAVEN_CACHE_FOLDER) -DskipTests -DskipITs'
integrationTestMavenGoal: 'verify'
skipDeploy: ${{ variables.SKIP_DEPLOY }}
skipTest: ${{ variables.SKIP_TESTS }}
......
......@@ -55,8 +55,8 @@ stages:
parameters:
mavenGoal: 'package'
mavenPublishJUnitResults: true
serviceCoreMavenOptions: '-P wks-core --settings .mvn/community-maven-ado.settings.xml'
mavenOptions: '-P wks-azure --settings .mvn/community-maven-ado.settings.xml -Dmaven.repo.local=$(MAVEN_CACHE_FOLDER)'
serviceCoreMavenOptions: '-pl wks-core --settings .mvn/community-maven-ado.settings.xml -Dmaven.repo.local=$(MAVEN_CACHE_FOLDER)'
mavenOptions: '-pl provider/wks-azure --settings .mvn/community-maven-ado.settings.xml -Dmaven.repo.local=$(MAVEN_CACHE_FOLDER)'
copyFileContents: |
pom.xml
provider/wks-azure/maven/settings.xml
......@@ -78,7 +78,7 @@ stages:
chartPath: ${{ variables.chartPath }}
valuesFile: ${{ variables.valuesFile }}
testCoreMavenPomFile: 'testing/wks-test-core/pom.xml'
testCoreMavenOptions: '--settings $(System.DefaultWorkingDirectory)/drop/.mvn/community-maven-ado.settings.xml -DskipTests -DskipITs'
testCoreMavenOptions: '--settings $(System.DefaultWorkingDirectory)/drop/.mvn/community-maven-ado.settings.xml -Dmaven.repo.local=$(MAVEN_CACHE_FOLDER) -DskipTests -DskipITs'
integrationTestMavenGoal: 'verify'
skipDeploy: ${{ variables.SKIP_DEPLOY }}
skipTest: ${{ variables.SKIP_TESTS }}
......
File suppressed by a .gitattributes entry or the file's encoding is unsupported.
package org.opengroup.osdu.wks.provider.azure;
import org.opengroup.osdu.wks.config.ThreadScopeBeanFactoryPostProcessor;
import org.opengroup.osdu.wks.provider.interfaces.SubscriptionManager;
import org.opengroup.osdu.wks.service.SchemaService;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.config.BeanFactoryPostProcessor;
import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication;
import org.springframework.context.ApplicationContext;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.ComponentScan;
import org.springframework.web.client.RestTemplate;
@SpringBootApplication
@ComponentScan(value = {
"org.opengroup.osdu"
})
public class WksServiceApplication {
private final static Logger LOGGER = LoggerFactory.getLogger(SchemaService.class);
public static void main(String[] args) {
ApplicationContext context = SpringApplication.run(org.opengroup.osdu.wks.WksServiceApplication.class, args);
try {
SubscriptionManager subscriptionManager = context.getBean(SubscriptionManager.class);
subscriptionManager.subscribeRecordsChangeEvent();
}
catch (Exception e)
{
LOGGER.error(e.getMessage(), e);
}
}
@Bean("restTemplate")
public RestTemplate restTemplate() {
return new RestTemplate();
}
@Bean
public static BeanFactoryPostProcessor beanFactoryPostProcessor() {
return new ThreadScopeBeanFactoryPostProcessor();
}
}
......@@ -60,7 +60,7 @@ public class SubscriptionManagerImpl implements SubscriptionManager {
new MessageHandlerOptions(Integer.parseUnsignedInt(azureBootstrapConfig.getMaxConcurrentCalls()),
false,
Duration.ofSeconds(Integer.parseUnsignedInt(azureBootstrapConfig.getMaxLockRenewDurationInSeconds())),
Duration.ofMillis(200)
Duration.ofSeconds(1)
),
executorService);
......
......@@ -53,13 +53,19 @@ spring.application.name=wks-azure
shared_tenant=${default_tenant}
# Storage service
STORAGE_API=${storage_service_endpoint}
osdu.host.storageServiceUrl=${storage_service_endpoint}
osdu.restclient.storageRetryIntervalInSeconds=1
osdu.restclient.storageRetryCounter=3
# Search service
SEARCH_API=${search_service_endpoint}
osdu.host.searchServiceUrl=${search_service_endpoint}
osdu.restclient.searchRetryIntervalInSeconds=1
osdu.restclient.searchRetryCounter=3
#Schema service
osdu.host.schemaServiceUrl=${schema_service_endpoint}
osdu.restclient.schemaRetryIntervalInSeconds=1
osdu.restclient.schemaRetryCounter=3
# Partition service
PARTITION_API=${partition_service_endpoint}
......
......@@ -24,6 +24,7 @@ public class Constants {
public static final String LAST_MODIFIED_DATE = "last_modified_date";
public static final String DASH = "-";
public static final String POINT = ".";
public static final String Slash = "/";
public static final String MISSING_RAW_RECORD_DETAILS = "Missing raw records details";
public static final String DATA_PARTITION_ID_NEEDED = "Data Partition Id is needed";
public static final String CORRELATION_ID_NEEDED = "Correlation Id is needed";
......
package org.opengroup.osdu.wks.constants;
public class SearchConstants {
public static final String POST_SEARCH_ENDPOINT = "/query";
}
package org.opengroup.osdu.wks.constants;
public class StorageConstants {
public static final String PUT_STORAGE_ENDPOINT = "/records";
public static final String GET_STORAGE_ENDPOINT = "/records";
}
package org.opengroup.osdu.wks.service;
import lombok.Getter;
import lombok.Setter;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.boot.context.properties.ConfigurationProperties;
import org.springframework.context.annotation.Configuration;
@Configuration
@ConfigurationProperties("osdu.restclient")
@Getter
@Setter
public class RestClientConfig {
@Value("#{new Integer('${searchRetryIntervalInSeconds:60}')}")
private Integer searchRetryIntervalInSeconds;
@Value("#{new Integer('${storageRetryIntervalInSeconds:60}')}")
private Integer storageRetryIntervalInSeconds;
@Value("#{new Integer('${schemaRetryIntervalInSeconds:60}')}")
private Integer schemaRetryIntervalInSeconds;
@Value("#{new Integer('${searchRetryCounter:3}')}")
private Integer searchRetryCounter;
@Value("#{new Integer('${storageRetryCounter:3}')}")
private Integer storageRetryCounter;
@Value("#{new Integer('${schemaRetryCounter:3}')}")
private Integer schemaRetryCounter;
}
......@@ -22,6 +22,7 @@ import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.http.HttpMethod;
import org.springframework.stereotype.Component;
import org.springframework.web.client.RestTemplate;
import java.net.URISyntaxException;
import java.util.ArrayList;
......@@ -33,23 +34,31 @@ public class SchemaService implements ISchemaService{
private final static Logger LOGGER = LoggerFactory.getLogger(SchemaService.class);
private static final String SCHEMA_SERVICE_ERROR_MESSAGE = "Schema service status code %s and message %s";
@Autowired
private RestClient restClient;
@Autowired
private UserCredential userCredential;
@Autowired
private ITenantFactory tenantFactory;
private final RestClient restClient;
private final ServiceHostConfiguration serviceHostConfiguration;
private final UserCredential userCredential;
private final ITenantFactory tenantFactory;
private final RequestIdentity requestIdentity;
private final KindUtil kindUtil;
@Autowired
private RequestIdentity requestIdentity;
@Autowired
private ServiceHostConfiguration serviceConfig;
@Autowired
private KindUtil kindUtil;
public SchemaService(
RestClient restClient,
RestClientConfig restClientConfig,
ServiceHostConfiguration serviceHostConfiguration,
KindUtil kindUtil,
RequestIdentity requestIdentity,
ITenantFactory tenantFactory,
UserCredential userCredential) {
this.restClient = restClient;
this.restClient.setRetryIntervalInSeconds(restClientConfig.getSchemaRetryIntervalInSeconds());
this.restClient.setRetryCounter(restClientConfig.getSchemaRetryCounter());
this.serviceHostConfiguration = serviceHostConfiguration;
this.kindUtil = kindUtil;
this.requestIdentity = requestIdentity;
this.tenantFactory = tenantFactory;
this.userCredential = userCredential;
}
public List<String> getSchemas(String kind) throws ApplicationException {
RequestParameters requestParameters = new RequestParameters();
......@@ -106,7 +115,7 @@ public class SchemaService implements ISchemaService{
try {
String[] kindArr = kind.split(Constants.COLON_SEPARATOR);
String majorVersion = kindUtil.getMajorVersionFromKind(kind).toString();
String schemaUrl = serviceConfig.getSchemaServiceUrl() + SchemaConstants.POST_SCHEMA_ENDPOINT;
String schemaUrl = serviceHostConfiguration.getSchemaServiceUrl().concat(SchemaConstants.POST_SCHEMA_ENDPOINT);
URIBuilder uriBuilder = new URIBuilder(schemaUrl);
uriBuilder.addParameter("authority", kindArr[0]);
uriBuilder.addParameter("source", kindArr[1]);
......
......@@ -4,13 +4,13 @@ import org.apache.http.HttpStatus;
import org.opengroup.osdu.core.common.model.tenant.TenantInfo;
import org.opengroup.osdu.core.common.provider.interfaces.ITenantFactory;
import org.opengroup.osdu.wks.config.RequestIdentity;
import org.opengroup.osdu.wks.constants.SearchConstants;
import org.opengroup.osdu.wks.exceptions.ApplicationException;
import org.opengroup.osdu.wks.model.RequestParameters;
import org.opengroup.osdu.wks.model.RestResponse;
import org.opengroup.osdu.wks.provider.interfaces.UserCredential;
import org.opengroup.osdu.wks.util.RestClient;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.http.HttpMethod;
import org.springframework.stereotype.Service;
......@@ -19,30 +19,33 @@ public class SearchService {
public static final String SEARCH_SERVICE_ERROR_MESSAGE = "Search service status code %s and message %s";
@Value("${SEARCH_API}/query")
private String searchServiceUrl;
private final RestClient restClient;
private final UserCredential userCredential;
private final ITenantFactory tenantStorageFactory;
private final RequestIdentity requestIdentity;
private final ServiceHostConfiguration serviceHostConfiguration;
@Autowired
public SearchService(
RestClient restClient,
UserCredential userCredential,
ITenantFactory tenantStorageFactory,
RequestIdentity requestIdentity) {
this.restClient = restClient;
this.userCredential = userCredential;
this.tenantStorageFactory = tenantStorageFactory;
this.requestIdentity = requestIdentity;
}
@Autowired
public SearchService(
RestClient restClient,
RestClientConfig restClientConfig,
ServiceHostConfiguration serviceHostConfiguration,
UserCredential userCredential,
ITenantFactory tenantStorageFactory,
RequestIdentity requestIdentity) {
this.restClient = restClient;
this.restClient.setRetryIntervalInSeconds(restClientConfig.getSearchRetryIntervalInSeconds());
this.restClient.setRetryCounter(restClientConfig.getSearchRetryCounter());
this.serviceHostConfiguration = serviceHostConfiguration;
this.userCredential = userCredential;
this.tenantStorageFactory = tenantStorageFactory;
this.requestIdentity = requestIdentity;
}
public String query(String payload) throws ApplicationException {
RequestParameters requestParameters = new RequestParameters();
TenantInfo tenantInfo = tenantStorageFactory.getTenantInfo(requestIdentity.getDataPartitionId());
requestParameters.setUrl(searchServiceUrl);
requestParameters.setUrl(serviceHostConfiguration.getSearchServiceUrl().concat(SearchConstants.POST_SEARCH_ENDPOINT));
requestParameters.setMethod(HttpMethod.POST);
requestParameters.setAppKey("");
requestParameters.setAuthKey(userCredential.getIdToken(tenantInfo));
......
......@@ -10,9 +10,7 @@ import org.springframework.context.annotation.Configuration;
@Getter
@Setter
class ServiceHostConfiguration {
private String searchServiceUrl;
private String storageServiceUrl;
private String schemaServiceUrl;
}
......@@ -7,6 +7,8 @@ import org.apache.http.HttpStatus;
import org.opengroup.osdu.core.common.model.tenant.TenantInfo;
import org.opengroup.osdu.core.common.provider.interfaces.ITenantFactory;
import org.opengroup.osdu.wks.config.RequestIdentity;
import org.opengroup.osdu.wks.constants.Constants;
import org.opengroup.osdu.wks.constants.StorageConstants;
import org.opengroup.osdu.wks.exceptions.ApplicationException;
import org.opengroup.osdu.wks.exceptions.AuthorizationException;
import org.opengroup.osdu.wks.model.RequestParameters;
......@@ -16,29 +18,37 @@ import org.opengroup.osdu.wks.util.RestClient;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.http.HttpMethod;
import org.springframework.stereotype.Service;
import org.springframework.web.client.RestTemplate;
@Service
public class StorageServiceImpl implements StorageService {
private final static Logger LOGGER = LoggerFactory.getLogger(StorageServiceImpl.class);
private final static String WKS_SUFFIX = "wks-source";
@Autowired
private RestClient restClient;
@Autowired
private UserCredential userCrendential;
private final RestClient restClient;
private final ServiceHostConfiguration serviceHostConfiguration;
private final UserCredential userCredential;
private final ITenantFactory tenantFactory;
private final RequestIdentity requestIdentity;
@Autowired
private ITenantFactory tenantFactory;
@Autowired
private RequestIdentity requestIdentity;
@Value("${STORAGE_API}/records")
private String storageServiceUrl;
public StorageServiceImpl(
RestClient restClient,
RestClientConfig restClientConfig,
ServiceHostConfiguration serviceHostConfiguration,
UserCredential userCredential,
RequestIdentity requestIdentity,
ITenantFactory tenantFactory) {
this.restClient = restClient;
this.restClient.setRetryIntervalInSeconds(restClientConfig.getStorageRetryIntervalInSeconds());
this.restClient.setRetryCounter(restClientConfig.getStorageRetryCounter());
this.serviceHostConfiguration = serviceHostConfiguration;
this.userCredential = userCredential;
this.requestIdentity = requestIdentity;
this.tenantFactory = tenantFactory;
}
/**
* This method returns the raw record for the given record id.
......@@ -50,10 +60,11 @@ public class StorageServiceImpl implements StorageService {
@Override
public Optional<String> getRecord(String id) {
String tenantName = requestIdentity.getDataPartitionId();
String getRecordUrl = serviceHostConfiguration.getStorageServiceUrl().concat(StorageConstants.GET_STORAGE_ENDPOINT).concat(Constants.Slash).concat(id);
RequestParameters requestParameters = new RequestParameters();
TenantInfo tenantInfo = tenantFactory.getTenantInfo(tenantName);
requestParameters.setUrl(storageServiceUrl + "/" + id);
requestParameters.setUrl(getRecordUrl);
requestParameters.setMethod(HttpMethod.GET);
requestParameters.setAuthKey(getAuthKey(tenantInfo));
requestParameters.setCorrelationId(requestIdentity.getCorrelationId());
......@@ -76,8 +87,10 @@ public class StorageServiceImpl implements StorageService {
@Override
public String putRecords(String[] jsonStrArr, String tenantName, String correlationId) throws ApplicationException {
RequestParameters requestParameters = new RequestParameters();
String putRecordUrl = serviceHostConfiguration.getStorageServiceUrl().concat(StorageConstants.PUT_STORAGE_ENDPOINT);
TenantInfo tenantInfo = tenantFactory.getTenantInfo(tenantName);
requestParameters.setUrl(storageServiceUrl);
requestParameters.setUrl(putRecordUrl);
requestParameters.setMethod(HttpMethod.PUT);
requestParameters.setAuthKey(getAuthKey(tenantInfo));
// For Storage service put operation payload should be list which is change into
......@@ -97,7 +110,7 @@ public class StorageServiceImpl implements StorageService {
private String getAuthKey(TenantInfo tenantInfo) {
try {
return userCrendential.getIdToken(tenantInfo);
return userCredential.getIdToken(tenantInfo);
} catch(ApplicationException e) {
LOGGER.warn(e.getErrorMsg(), e);
throw new AuthorizationException(e);
......
......@@ -120,7 +120,7 @@ public class WKSServiceImpl implements WKSService {
private void transformRecords(RawRecordDetails[] rawRecordsDetails, Map<String, List<MappingsModel>> mappingsMap,
String dataPartitionId, String correlationId) throws ApplicationException, BadRequestException {
String dataPartitionId, String correlationId) throws ApplicationException {
List<Tree> wksRecordTreeList = new ArrayList<>();
List<RelationshipStatus> relationshipStatusList = new ArrayList<>();
for (RawRecordDetails rawRecord : rawRecordsDetails) {
......@@ -149,18 +149,27 @@ public class WKSServiceImpl implements WKSService {
catch (ApplicationException e) {
LOGGER.warn(Constants.TRANSFORMATION_FAILED_FOR_GIVEN_WKS_KIND, rawRecord.getId(), mappings.getTargetSchemaKind(), e.getErrorMsg(), e);
}
catch (Exception e) {
LOGGER.warn(Constants.TRANSFORMATION_FAILED_FOR_GIVEN_WKS_KIND, rawRecord.getId(), mappings.getTargetSchemaKind(), e.getMessage(), e);
}
}
} catch (ApplicationException e) {
LOGGER.warn(Constants.TRANFORMATION_FAILED, rawRecord.getId(), e.getErrorMsg(), e);
}
catch (Exception e) {
LOGGER.info(Constants.TRANFORMATION_FAILED, rawRecord.getId(), e.getMessage(), e);
}
}
if (!wksRecordTreeList.isEmpty()) {
saveAllWksRecords(wksRecordTreeList, dataPartitionId, correlationId);
LOGGER.info("Successfully saved {} WKS records in storage", wksRecordTreeList.size());
if(!relationshipStatusList.isEmpty()) {
statusStoreService.createOrUpdateStoreEntry(relationshipStatusList);
LOGGER.info("Successfully saved relationship status for records for which WKS were not found");
}
}
}
......
......@@ -68,8 +68,7 @@ public class RelationshipBlockHandler {
List<String> ids = relationship.getIds();
rawIds.addAll(ids);
}
List<Relationship> wksRelationships = fetchWksRelationships(rawRelationships);
List<Relationship> wksRelationships = fetchWksRelationships(rawRelationships);
Iterator<Relationship> relationshipIterator = rawRelationships.iterator();
Iterator<Relationship> wksRelationshipIterator = wksRelationships.iterator();
Set<String> pendingEntitiesToBeAdded = new HashSet<>();
......@@ -91,46 +90,46 @@ public class RelationshipBlockHandler {
return buildUpdatedRelationshipBlock(rawRelationshipsBlock, pendingEntitiesToBeAdded, pendingIdsToBeAdded);
}
private UpdatedRelationshipBlock buildUpdatedRelationshipBlock(RelationshipsBlock rawRelationshipsBlock,
Set<String> pendingEntitiesToBeAdded, List<String> pendingIdsToBeAdded) {
UpdatedRelationshipBlock updatedRelationshipBlock = new UpdatedRelationshipBlock();
private UpdatedRelationshipBlock buildUpdatedRelationshipBlock(RelationshipsBlock rawRelationshipsBlock,
Set<String> pendingEntitiesToBeAdded, List<String> pendingIdsToBeAdded) {
UpdatedRelationshipBlock updatedRelationshipBlock = new UpdatedRelationshipBlock();
updatedRelationshipBlock.setRelationshipTree(rawRelationshipsBlock.getRelationshipBlockAsTree());
updatedRelationshipBlock.setPendingIds(pendingIdsToBeAdded);
updatedRelationshipBlock.setPendingEnitites(pendingEntitiesToBeAdded);
return updatedRelationshipBlock;
}
}
private List<Relationship> fetchWksRelationships(List<Relationship> rawRelationships) {
List<Relationship> wksRelationships = new ArrayList<>();
private List<Relationship> fetchWksRelationships(List<Relationship> rawRelationships) {
List<Relationship> wksRelationships = new ArrayList<>();
for(Relationship rawRelationship : rawRelationships) {
Relationship wksRelationship = null;
try {
wksRelationship = getRelationshipBlockWithWksIds(rawRelationship);
} catch (ApplicationException e) {
LOGGER.info(e.getMessage(), e);
}
if(wksRelationship != null) {
wksRelationships.add(wksRelationship);
}
Relationship wksRelationship = null;
try {
wksRelationship = getRelationshipBlockWithWksIds(rawRelationship);
} catch (ApplicationException e) {
LOGGER.info(e.getMessage(), e);
}
if(wksRelationship != null) {
wksRelationships.add(wksRelationship);
}
}
return wksRelationships;
}
return wksRelationships;
}
private Relationship getRelationshipBlockWithWksIds(Relationship rawRelationship) throws ApplicationException {
List<String> rawIds = rawRelationship.getIds();
List<String> wksIds = getWksIds(rawIds);
private Relationship getRelationshipBlockWithWksIds(Relationship rawRelationship) throws ApplicationException {
List<String> rawIds = rawRelationship.getIds();
List<String> wksIds = getWksIds(rawIds);
rawRelationship.setIds(wksIds);
return rawRelationship;
}
rawRelationship.setIds(wksIds);
return rawRelationship;
}
private List<String> getWksIds(List<String> rawIds) {
return rawIds.stream().map(this::getWksId).collect(Collectors.toList());
}