There is a security vulnerability in SSH key-generation using GitKraken < v8.0.1. If you used this tool to create SSH keys, please update GitKraken and regenerate. If you need help with this, contact forum-support@opengroup.org

Commit 72933199 authored by David Diederich's avatar David Diederich
Browse files

Merge remote-tracking branch 'origin/master' into release/0.3

parents 2e5833fd e9d821aa
Pipeline #2994 passed with stages
in 29 minutes and 54 seconds
......@@ -317,9 +317,10 @@ The following software have components provided under the terms of this license:
- Microsoft Application Insights Java SDK Web Module (from https://github.com/Microsoft/ApplicationInsights-Java)
- Microsoft Application Insights Log4j 2 Appender (from https://github.com/Microsoft/ApplicationInsights-Java)
- Microsoft Application Insights Logback Appender (from https://github.com/Microsoft/ApplicationInsights-Java)
- Mockito (from http://www.mockito.org)
- Mockito (from http://mockito.org)
- Mockito (from http://mockito.org)
- Mockito (from http://mockito.org)
- Mockito (from http://www.mockito.org)
- Mojo's Maven plugin for Cobertura (from http://mojo.codehaus.org/cobertura-maven-plugin/)
- Netty Reactive Streams Implementation (from )
- Netty/Buffer (from http://netty.io/)
......@@ -856,9 +857,10 @@ The following software have components provided under the terms of this license:
- Microsoft Azure client library for Identity (from https://github.com/Azure/azure-sdk-for-java)
- Microsoft Azure client library for KeyVault Secrets (from https://github.com/Azure/azure-sdk-for-java)
- Microsoft Azure common module for Storage (from https://github.com/Azure/azure-sdk-for-java)
- Mockito (from http://mockito.org)
- Mockito (from http://www.mockito.org)
- Mockito (from http://mockito.org)
- Mockito (from http://mockito.org)
- Mockito (from http://mockito.org)
- Netty/Codec/HTTP (from )
- Netty/Common (from )
- Plexus :: Default Container (from )
......
......@@ -9,7 +9,7 @@
</parent>
<artifactId>indexer-core</artifactId>
<version>1.0.4-SNAPSHOT</version>
<version>1.0.5-SNAPSHOT</version>
<name>indexer-core</name>
<description>Indexer Service Core</description>
<packaging>jar</packaging>
......@@ -18,7 +18,6 @@
<dependency>
<groupId>org.opengroup.osdu</groupId>
<artifactId>os-core-common</artifactId>
<version>0.0.13</version>
</dependency>
<!-- spring boot dependencies -->
......
// Copyright 2017-2019, Schlumberger
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package org.opengroup.osdu.indexer.api;
import com.google.common.reflect.TypeToken;
import com.google.gson.Gson;
import com.google.gson.JsonParseException;
import io.swagger.annotations.ApiOperation;
import lombok.extern.java.Log;
import org.opengroup.osdu.core.common.model.http.DpsHeaders;
import org.opengroup.osdu.core.common.model.http.AppException;
import org.opengroup.osdu.core.common.model.search.RecordChangedMessages;
import org.opengroup.osdu.indexer.SwaggerDoc;
import org.opengroup.osdu.core.common.model.indexer.JobStatus;
import org.opengroup.osdu.core.common.model.indexer.RecordReindexRequest;
import org.opengroup.osdu.indexer.service.IndexerService;
import org.opengroup.osdu.indexer.service.ReindexService;
import org.opengroup.osdu.core.common.model.indexer.RecordInfo;
import org.springframework.http.HttpStatus;
import org.springframework.http.ResponseEntity;
import org.springframework.web.bind.annotation.*;
import org.springframework.web.context.annotation.RequestScope;
import javax.inject.Inject;
import javax.validation.Valid;
import javax.validation.constraints.NotNull;
import java.lang.reflect.Type;
import java.util.List;
@Log
@RestController
@RequestMapping("/_dps/task-handlers")
@RequestScope
public class RecordIndexerApi {
@Inject
private IndexerService indexerService;
@Inject
private ReindexService reIndexService;
// THIS IS AN INTERNAL USE API ONLY
// THAT MEANS WE DON'T DOCUMENT IT IN SWAGGER, ACCESS IS LIMITED TO CLOUD TASK QUEUE CALLS ONLY
@PostMapping(path = "/index-worker", consumes = "application/json")
@ApiOperation(hidden = true, value = "", notes = "")
public ResponseEntity<JobStatus> indexWorker (
@NotNull(message = SwaggerDoc.REQUEST_VALIDATION_NOT_NULL_BODY)
@Valid @RequestBody RecordChangedMessages recordChangedMessages) throws Exception {
if (recordChangedMessages.missingAccountId()) {
throw new AppException(org.apache.http.HttpStatus.SC_BAD_REQUEST, "Invalid tenant",
String.format("Required header: '%s' not found", DpsHeaders.DATA_PARTITION_ID));
}
try {
if (recordChangedMessages == null) {
log.info("record change messages is null");
}
Type listType = new TypeToken<List<RecordInfo>>() {
}.getType();
List<RecordInfo> recordInfos = new Gson().fromJson(recordChangedMessages.getData(), listType);
if (recordInfos.size() == 0) {
log.info("none of record-change message can be deserialized");
return new ResponseEntity(HttpStatus.OK);
}
this.indexerService.processRecordChangedMessages(recordChangedMessages, recordInfos);
return new ResponseEntity(HttpStatus.OK);
} catch (AppException e) {
throw e;
} catch (JsonParseException e) {
throw new AppException(org.apache.http.HttpStatus.SC_BAD_REQUEST, "Request payload parsing error", "Unable to parse request payload.", e);
} catch (Exception e) {
throw new AppException(org.apache.http.HttpStatus.SC_BAD_REQUEST, "Unknown error", "An unknown error has occurred.", e);
}
}
// THIS IS AN INTERNAL USE API ONLY
// THAT MEANS WE DON'T DOCUMENT IT IN SWAGGER, ACCESS IS LIMITED TO CLOUD TASK QUEUE CALLS ONLY
@PostMapping("/reindex-worker")
@ApiOperation(hidden = true, value = "", notes = "")
public ResponseEntity reindex(
@RequestBody @NotNull(message = SwaggerDoc.REQUEST_VALIDATION_NOT_NULL_BODY)
@Valid RecordReindexRequest recordReindexRequest) {
return new ResponseEntity(reIndexService.reindexRecords(recordReindexRequest),HttpStatus.OK);
}
}
\ No newline at end of file
// Copyright 2017-2019, Schlumberger
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package org.opengroup.osdu.indexer.api;
import com.google.common.reflect.TypeToken;
import com.google.gson.Gson;
import com.google.gson.JsonParseException;
import io.swagger.annotations.ApiOperation;
import lombok.extern.java.Log;
import org.opengroup.osdu.core.common.model.http.DpsHeaders;
import org.opengroup.osdu.core.common.model.http.AppException;
import org.opengroup.osdu.core.common.model.search.RecordChangedMessages;
import org.opengroup.osdu.indexer.SwaggerDoc;
import org.opengroup.osdu.core.common.model.indexer.JobStatus;
import org.opengroup.osdu.core.common.model.indexer.RecordReindexRequest;
import org.opengroup.osdu.indexer.service.IndexerService;
import org.opengroup.osdu.indexer.service.ReindexService;
import org.opengroup.osdu.core.common.model.indexer.RecordInfo;
import org.springframework.http.HttpStatus;
import org.springframework.http.ResponseEntity;
import org.springframework.web.bind.annotation.*;
import org.springframework.web.context.annotation.RequestScope;
import javax.inject.Inject;
import javax.validation.Valid;
import javax.validation.constraints.NotNull;
import java.lang.reflect.Type;
import java.util.List;
@Log
@RestController
@RequestMapping("/_dps/task-handlers")
@RequestScope
public class RecordIndexerApi {
@Inject
private IndexerService indexerService;
@Inject
private ReindexService reIndexService;
// THIS IS AN INTERNAL USE API ONLY
// THAT MEANS WE DON'T DOCUMENT IT IN SWAGGER, ACCESS IS LIMITED TO CLOUD TASK QUEUE CALLS ONLY
@PostMapping(path = "/index-worker", consumes = "application/json")
@ApiOperation(hidden = true, value = "", notes = "")
public ResponseEntity<JobStatus> indexWorker (
@NotNull(message = SwaggerDoc.REQUEST_VALIDATION_NOT_NULL_BODY)
@Valid @RequestBody RecordChangedMessages recordChangedMessages) throws Exception {
if (recordChangedMessages.missingAccountId()) {
throw new AppException(org.apache.http.HttpStatus.SC_BAD_REQUEST, "Invalid tenant",
String.format("Required header: '%s' not found", DpsHeaders.DATA_PARTITION_ID));
}
try {
if (recordChangedMessages == null) {
log.info("record change messages is null");
}
Type listType = new TypeToken<List<RecordInfo>>() {
}.getType();
List<RecordInfo> recordInfos = new Gson().fromJson(recordChangedMessages.getData(), listType);
if (recordInfos.size() == 0) {
log.info("none of record-change message can be deserialized");
return new ResponseEntity(HttpStatus.OK);
}
this.indexerService.processRecordChangedMessages(recordChangedMessages, recordInfos);
return new ResponseEntity(HttpStatus.OK);
} catch (AppException e) {
throw e;
} catch (JsonParseException e) {
throw new AppException(org.apache.http.HttpStatus.SC_BAD_REQUEST, "Request payload parsing error", "Unable to parse request payload.", e);
} catch (Exception e) {
throw new AppException(org.apache.http.HttpStatus.SC_BAD_REQUEST, "Unknown error", "An unknown error has occurred.", e);
}
}
// THIS IS AN INTERNAL USE API ONLY
// THAT MEANS WE DON'T DOCUMENT IT IN SWAGGER, ACCESS IS LIMITED TO CLOUD TASK QUEUE CALLS ONLY
@PostMapping("/reindex-worker")
@ApiOperation(hidden = true, value = "", notes = "")
public ResponseEntity<?> reindex(
@RequestBody @NotNull(message = SwaggerDoc.REQUEST_VALIDATION_NOT_NULL_BODY)
@Valid RecordReindexRequest recordReindexRequest) {
return new ResponseEntity<>(reIndexService.reindexRecords(recordReindexRequest, false), HttpStatus.OK);
}
}
// Copyright 2017-2019, Schlumberger
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package org.opengroup.osdu.indexer.api;
import org.opengroup.osdu.core.common.model.search.SearchServiceRole;
import org.opengroup.osdu.indexer.logging.AuditLogger;
import org.opengroup.osdu.core.common.model.indexer.RecordReindexRequest;
import org.opengroup.osdu.indexer.service.ReindexService;
import org.springframework.http.ResponseEntity;
import org.springframework.security.access.prepost.PreAuthorize;
import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.RequestBody;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RestController;
import org.springframework.web.context.annotation.RequestScope;
import javax.inject.Inject;
import javax.validation.Valid;
import javax.validation.constraints.NotNull;
import static java.util.Collections.singletonList;
@RestController
@RequestMapping("/reindex")
@RequestScope
public class ReindexApi {
@Inject
private ReindexService reIndexService;
@Inject
private AuditLogger auditLogger;
@PreAuthorize("@authorizationFilter.hasPermission('" + SearchServiceRole.ADMIN + "')")
@PostMapping
public ResponseEntity reindex(
@NotNull @Valid @RequestBody RecordReindexRequest recordReindexRequest) {
this.reIndexService.reindexRecords(recordReindexRequest);
this.auditLogger.getReindex(singletonList(recordReindexRequest.getKind()));
return new ResponseEntity (org.springframework.http.HttpStatus.OK);
}
}
\ No newline at end of file
// Copyright 2017-2019, Schlumberger
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package org.opengroup.osdu.indexer.api;
import org.opengroup.osdu.core.common.model.search.SearchServiceRole;
import org.opengroup.osdu.indexer.logging.AuditLogger;
import org.opengroup.osdu.core.common.model.indexer.RecordReindexRequest;
import org.opengroup.osdu.indexer.service.IndexSchemaService;
import org.opengroup.osdu.indexer.service.ReindexService;
import org.springframework.http.ResponseEntity;
import org.springframework.security.access.prepost.PreAuthorize;
import org.springframework.web.bind.annotation.*;
import org.springframework.web.context.annotation.RequestScope;
import javax.inject.Inject;
import javax.validation.Valid;
import javax.validation.constraints.NotNull;
import java.io.IOException;
import static java.util.Collections.singletonList;
@RestController
@RequestMapping("/reindex")
@RequestScope
public class ReindexApi {
@Inject
private ReindexService reIndexService;
@Inject
private IndexSchemaService indexSchemaService;
@Inject
private AuditLogger auditLogger;
@PreAuthorize("@authorizationFilter.hasPermission('" + SearchServiceRole.ADMIN + "')")
@PostMapping
public ResponseEntity<?> reindex(
@NotNull @Valid @RequestBody RecordReindexRequest recordReindexRequest,
@RequestParam(value = "force_clean", defaultValue = "false") boolean forceClean) throws IOException {
this.reIndexService.reindexRecords(recordReindexRequest, this.indexSchemaService.isStorageSchemaSyncRequired(recordReindexRequest.getKind(), forceClean));
this.auditLogger.getReindex(singletonList(recordReindexRequest.getKind()));
return new ResponseEntity<>(org.springframework.http.HttpStatus.OK);
}
}
// Copyright 2017-2019, Schlumberger
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package org.opengroup.osdu.indexer.service;
import com.google.common.base.Strings;
import com.google.gson.Gson;
import com.google.gson.JsonSyntaxException;
import com.google.gson.internal.LinkedTreeMap;
import com.google.gson.reflect.TypeToken;
import org.apache.http.HttpStatus;
import org.opengroup.osdu.core.common.model.indexer.IndexingStatus;
import org.opengroup.osdu.core.common.model.indexer.JobStatus;
import org.opengroup.osdu.indexer.util.parser.DateTimeParser;
import org.opengroup.osdu.indexer.util.parser.GeoShapeParser;
import org.opengroup.osdu.indexer.util.parser.NumberParser;
import org.springframework.stereotype.Service;
import org.springframework.web.context.annotation.RequestScope;
import javax.inject.Inject;
import java.lang.reflect.Type;
import java.util.Map;
@Service
@RequestScope
public class AttributeParsingServiceImpl implements IAttributeParsingService {
private static final String GEOJSON = "GeoJSON";
@Inject
private NumberParser numberParser;
@Inject
private DateTimeParser dateTimeParser;
@Inject
private GeoShapeParser geoShapeParser;
@Inject
private GeometryConversionService geometryConversionService;
@Inject
private JobStatus jobStatus;
@Override
public void tryParseInteger(String recordId, String attributeName, Object attributeVal, Map<String, Object> dataMap) {
try {
int parsedInteger = this.numberParser.parseInteger(attributeName, attributeVal);
dataMap.put(attributeName, parsedInteger);
} catch (IllegalArgumentException e) {
jobStatus.addOrUpdateRecordStatus(recordId, IndexingStatus.WARN, HttpStatus.SC_BAD_REQUEST, e.getMessage(), String.format("record-id: %s | %s", recordId, e.getMessage()));
}
}
@Override
public void tryParseLong(String recordId, String attributeName, Object attributeVal, Map<String, Object> dataMap) {
try {
long parsedLong = this.numberParser.parseLong(attributeName, attributeVal);
dataMap.put(attributeName, parsedLong);
} catch (IllegalArgumentException e) {
jobStatus.addOrUpdateRecordStatus(recordId, IndexingStatus.WARN, HttpStatus.SC_BAD_REQUEST, e.getMessage(), String.format("record-id: %s | %s", recordId, e.getMessage()));
}
}
@Override
public void tryParseFloat(String recordId, String attributeName, Object attributeVal, Map<String, Object> dataMap) {
try {
float parsedFloat = this.numberParser.parseFloat(attributeName, attributeVal);
dataMap.put(attributeName, parsedFloat);
} catch (IllegalArgumentException e) {
jobStatus.addOrUpdateRecordStatus(recordId, IndexingStatus.WARN, HttpStatus.SC_BAD_REQUEST, e.getMessage(), String.format("record-id: %s | %s", recordId, e.getMessage()));
}
}
@Override
public void tryParseDouble(String recordId, String attributeName, Object attributeVal, Map<String, Object> dataMap) {
try {
double parsedDouble = this.numberParser.parseDouble(attributeName, attributeVal);
dataMap.put(attributeName, parsedDouble);
} catch (IllegalArgumentException e) {
jobStatus.addOrUpdateRecordStatus(recordId, IndexingStatus.WARN, HttpStatus.SC_BAD_REQUEST, e.getMessage(), String.format("record-id: %s | %s", recordId, e.getMessage()));
}
}
@Override
public void tryParseBoolean(String recordId, String attributeName, Object attributeVal, Map<String, Object> dataMap) {
String val = attributeVal == null ? null : String.valueOf(attributeVal);
dataMap.put(attributeName, Boolean.parseBoolean(val));
}
@Override
public void tryParseDate(String recordId, String attributeName, Object attributeVal, Map<String, Object> dataMap) {
String val = attributeVal == null ? null : String.valueOf(attributeVal);
if (Strings.isNullOrEmpty(val)) {
// skip indexing
return;
}
String utcDate = this.dateTimeParser.convertDateObjectToUtc(val);
if (Strings.isNullOrEmpty(utcDate)) {
String parsingError = String.format("datetime parsing error: unknown format for attribute: %s | value: %s", attributeName, attributeVal);
jobStatus.addOrUpdateRecordStatus(recordId, IndexingStatus.WARN, HttpStatus.SC_BAD_REQUEST, parsingError, String.format("record-id: %s | %s", recordId, parsingError));
} else {
dataMap.put(attributeName, utcDate);
}
}
@Override
public void tryParseGeopoint(String recordId, String attributeName, Map<String, Object> storageRecordData, Map<String, Object> dataMap) {
Object attributeVal = storageRecordData.get(attributeName);
try {
Type type = new TypeToken<Map<String, Double>>() {}.getType();
Map<String, Double> positionMap = new Gson().fromJson(attributeVal.toString(), type);
if (positionMap == null || positionMap.isEmpty()) return;
Map<String, Double> position = this.geometryConversionService.tryGetGeopoint(positionMap);
if (position == null || position.isEmpty()) return;
dataMap.put(attributeName, position);
// check if geo shape is not there and if it is not then create it in the schema as well as create the data.
LinkedTreeMap<String, Object> map = (LinkedTreeMap) storageRecordData.get(GEOJSON);
if (map == null || map.isEmpty()) {
Map<String, Object> geometry = this.geometryConversionService.getGeopointGeoJson(positionMap);
if (geometry == null) return;
dataMap.put(DATA_GEOJSON_TAG, geometry);
}
} catch (JsonSyntaxException | IllegalArgumentException e) {
String parsingError = String.format("geopoint parsing error: %s attribute: %s | value: %s", e.getMessage(), attributeName, attributeVal);
jobStatus.addOrUpdateRecordStatus(recordId, IndexingStatus.WARN, HttpStatus.SC_BAD_REQUEST, parsingError, String.format("record-id: %s | %s", recordId, parsingError));
}
}
@Override
public void tryParseGeojson(String recordId, String attributeName, Object attributeVal, Map<String, Object> dataMap) {
try {
Type type = new TypeToken<Map<String, Object>>() {}.getType();
Map<String, Object> geoJsonMap = new Gson().fromJson(attributeVal.toString(), type);
if (geoJsonMap == null || geoJsonMap.isEmpty()) return;
this.geoShapeParser.parseGeoJson(geoJsonMap);
dataMap.put(attributeName, geoJsonMap);
} catch (JsonSyntaxException | IllegalArgumentException e) {
String parsingError = String.format("geo-json shape parsing error: %s attribute: %s", e.getMessage(), attributeName);
jobStatus.addOrUpdateRecordStatus(recordId, IndexingStatus.WARN, HttpStatus.SC_BAD_REQUEST, parsingError, String.format("record-id: %s | %s", recordId, parsingError));
}
}
}
// Copyright 2017-2019, Schlumberger
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package org.opengroup.osdu.indexer.service;
import com.google.common.base.Strings;
import com.google.gson.Gson;
import com.google.gson.JsonSyntaxException;
import com.google.gson.internal.LinkedTreeMap;
import com.google.gson.reflect.TypeToken;
import org.apache.http.HttpStatus;
import org.opengroup.osdu.core.common.model.indexer.ElasticType;
import org.opengroup.osdu.core.common.model.indexer.IndexSchema;
import org.opengroup.osdu.core.common.model.indexer.IndexingStatus;
import org.opengroup.osdu.core.common.model.indexer.JobStatus;
import org.opengroup.osdu.core.common.Constants;
import org.opengroup.osdu.indexer.util.parser.BooleanParser;
import org.opengroup.osdu.indexer.util.parser.DateTimeParser;
import org.opengroup.osdu.indexer.util.parser.GeoShapeParser;
import org.opengroup.osdu.indexer.util.parser.NumberParser;
import org.springframework.stereotype.Service;
import org.springframework.web.context.annotation.RequestScope;
import javax.inject.Inject;
import java.lang.reflect.Array;
import java.lang.reflect.Type;
import java.util.*;
import java.util.function.BiFunction;
@Service
@RequestScope
public class AttributeParsingServiceImpl implements IAttributeParsingService {
private static final String GEOJSON = "GeoJSON";