There is a security vulnerability in SSH key-generation using GitKraken < v8.0.1. If you used this tool to create SSH keys, please update GitKraken and regenerate. If you need help with this, contact forum-support@opengroup.org

Commit 7790787a authored by Aalekh Jain's avatar Aalekh Jain
Browse files

Merge branch 'master' of...

Merge branch 'master' of https://community.opengroup.org/osdu/platform/system/search-service into master
parents 0e795e46 0b12ed30
Pipeline #46109 failed with stages
in 20 seconds
......@@ -36,3 +36,7 @@ analyze:
type: mvn
target: provider/search-ibm/pom.xml
path: .
- name: search-reference
type: mvn
target: provider/search-reference/pom.xml
path: .
......@@ -58,3 +58,5 @@ load-tests/*.pyc
### IntelliJ IDEA ###
*.iml
dist/
......@@ -14,6 +14,11 @@ variables:
GCP_DOMAIN: cloud.slb-ds.com
GCP_STORAGE_URL: https://osdu-search-dot-opendes.appspot.com/api/storage/v2/
# --- osdu gcp specific variables for cloudrun ---
OSDU_GCP_SERVICE: search
OSDU_GCP_VENDOR: gcp
OSDU_GCP_ENV_VARS: ENVIRONMENT=$OSDU_GCP_SPRING_PROFILES_ACTIVE,GOOGLE_CLOUD_PROJECT=$OSDU_GCP_PROJECT,REDIS_SEARCH_HOST=$REDIS_SEARCH_HOST,REDIS_SEARCH_PORT=$OSDU_GCP_REDIS_SEARCH_PORT,REDIS_GROUP_HOST=$REDIS_GROUP_HOST,INDEXER_HOST=$OSDU_GCP_INDEXER_HOST_SEARCH,AUTHORIZE_API=$OSDU_GCP_AUTHORIZE_API,ENTITLEMENTS_HOST=$OSDU_GCP_ENTITLEMENTS_URL,SECURITY_HTTPS_CERTIFICATE_TRUST=$OSDU_SECURITY_HTTPS_CERTIFICATE_TRUST,GOOGLE_AUDIENCES=$GOOGLE_AUDIENCE,PARTITION_API=$OSDU_GCP_PARTITION_API,POLICY_API=$OSDU_GCP_POLICY_API --vpc-connector=$OSDU_GCP_VPC_CONNECTOR
# Variables should be partially removed after migration on cloudrun gcp
OSDU_GCP_BUILD_SUBDIR: provider/search-gcp
OSDU_GCP_INT_TEST_SUBDIR: testing/integration-tests/search-test-gcp
OSDU_GCP_APPLICATION_NAME: os-search
......@@ -21,9 +26,12 @@ variables:
OSDU_GCP_TENANT_NAME: osdu
OSDU_GCP_STORAGE_URL: https://os-storage-dot-nice-etching-277309.uc.r.appspot.com/api/storage/v2/
OSDU_SECURITY_HTTPS_CERTIFICATE_TRUST: 'true'
OSDU_GCP_TEST_SUBDIR: testing/integration-tests/$OSDU_GCP_SERVICE-test-$OSDU_GCP_VENDOR
OSDU_GCP_HELM_PACKAGE_CHARTS: "devops/gcp/deploy devops/gcp/configmap"
IBM_BUILD_SUBDIR: provider/search-ibm
IBM_INT_TEST_SUBDIR: testing/integration-tests/search-test-ibm
DEFAULT_DATA_PARTITION_ID_TENANT1_IBM: tenant1
AZURE_SERVICE: search
AZURE_BUILD_SUBDIR: provider/search-azure
......@@ -38,7 +46,7 @@ include:
file: "build/maven.yml"
- project: "osdu/platform/ci-cd-pipelines"
file: "scanners/fossa.yml"
file: "scanners/fossa-maven.yml"
- project: "osdu/platform/ci-cd-pipelines"
file: "scanners/gitlab-ultimate.yml"
......@@ -53,9 +61,11 @@ include:
file: "cloud-providers/azure.yml"
- project: 'osdu/platform/ci-cd-pipelines'
ref: "master"
file: 'cloud-providers/osdu-gcp.yml'
file: 'cloud-providers/osdu-gcp-cloudrun.yml'
aws-test-java:
tags: ['aws-internal-test']
\ No newline at end of file
tags: ['aws-internal-test']
osdu-gcp-test:
variables:
OSDU_GCP_TESTING_DIR: "testing/integration-tests"
......@@ -7,7 +7,13 @@
<activeByDefault>true</activeByDefault>
</activation>
<properties>
<gitlab-server>community-maven-via-job-token</gitlab-server>
<repo.releases.id>community-maven-repo</repo.releases.id>
<publish.snapshots.id>community-maven-via-job-token</publish.snapshots.id>
<publish.releases.id>community-maven-via-job-token</publish.releases.id>
<repo.releases.url>https://community.opengroup.org/api/v4/groups/17/-/packages/maven</repo.releases.url>
<publish.snapshots.url>https://community.opengroup.org/api/v4/projects/19/packages/maven</publish.snapshots.url>
<publish.releases.url>https://community.opengroup.org/api/v4/projects/19/packages/maven</publish.releases.url>
</properties>
</profile>
......@@ -17,12 +23,18 @@
based authentication -->
<id>GitLab-Authenticate-With-Private-Token</id>
<activation>
<property>
<name>env.COMMUNITY_MAVEN_TOKEN</name>
</property>
<property>
<name>env.COMMUNITY_MAVEN_TOKEN</name>
</property>
</activation>
<properties>
<gitlab-server>community-maven-via-private-token</gitlab-server>
<repo.releases.id>community-maven-repo</repo.releases.id>
<publish.snapshots.id>community-maven-via-private-token</publish.snapshots.id>
<publish.releases.id>community-maven-via-private-token</publish.releases.id>
<repo.releases.url>https://community.opengroup.org/api/v4/groups/17/-/packages/maven</repo.releases.url>
<publish.snapshots.url>https://community.opengroup.org/api/v4/projects/19/packages/maven</publish.snapshots.url>
<publish.releases.url>https://community.opengroup.org/api/v4/projects/19/packages/maven</publish.releases.url>
</properties>
</profile>
</profiles>
......
This diff is collapsed.
......@@ -66,4 +66,6 @@ go-swagger brings to the go community a complete suite of fully-featured, high-p
indexpattern is defined in web.xml (in indexer) file with an environment variable as CRON_INDEX_CLEANUP_PATTERN
The scheduling of cron is done in the following repository:
https://slb-swt.visualstudio.com/data-management/_git/deployment-init-scripts?path=%2F3_post_deploy%2F1_appengine_cron%2Fcron.yaml&version=GBmaster
\ No newline at end of file
#New Update2
pipeline {
agent {
kubernetes {
cloud 'openshift'
label 'maven-openjdk18'
yaml """
spec:
containers:
- name: jnlp
image: quay.io/openshift/origin-jenkins-agent-maven:v4.0.0
volumeMounts:
- mountPath: "/tmp"
name: "workspace-volume"
readOnly: false
workingDir: "/tmp"
securityContext:
privileged: false
tty: false
resources:
limits:
cpu: 200m
memory: 2Gi
requests:
cpu: 200m
memory: 2Gi
restartPolicy: "Never"
"""
}
}
environment {
//Cluster environment variable(CLS_ENV). Like QA, DEV, PERF, PROD etc.
CLS_ENV = "dev"
//Service variable(CORE_SERVICE). Like indexer, search, delivery, storage, legal etc.
CORE_SERVICE = "search"
//GitHub repo URL credential ID for Environment variable files which saved as Secure text in Jenkins Credential.
GIT_ENV_VAR_PATH_URL = credentials('GitRepo-URL-For-Environment-variables')
//Personal token variable ID which saved as Secure text in Jenkins Credential. Like: GitHub-PRIVATE-TOKEN.
PRIVATE_TOKEN = credentials('GitHub-PRIVATE-TOKEN')
def runShell = sh (returnStdout: true, script: "curl --header 'PRIVATE-TOKEN: $PRIVATE_TOKEN' ''$GIT_ENV_VAR_PATH_URL'%2F'$CORE_SERVICE'_'$CLS_ENV'_env.json/raw?ref=master' -s -o env.json")
}
stages {
stage('Integration_test') {
environment {
def readContent = readJSON file: 'env.json'
AUTH_USER_ACCESS = "${readContent['AUTH_USER_ACCESS']}"
AUTH_USER_ACCESS_PASSWORD = "${readContent['AUTH_USER_ACCESS_PASSWORD']}"
DEFAULT_DATA_PARTITION_ID_TENANT1 = "${readContent['DEFAULT_DATA_PARTITION_ID_TENANT1']}"
DEFAULT_DATA_PARTITION_ID_TENANT2 = "${readContent['DEFAULT_DATA_PARTITION_ID_TENANT2']}"
DEPLOY_ENV = "${readContent['DEPLOY_ENV']}"
DOMAIN = "${readContent['DOMAIN']}"
ELASTIC_HOST = "${readContent['ELASTIC_HOST']}"
ELASTIC_PASSWORD = "${readContent['ELASTIC_PASSWORD']}"
ELASTIC_PORT = "${readContent['ELASTIC_PORT']}"
ELASTIC_USER_NAME = "${readContent['ELASTIC_USER_NAME']}"
ENTITLEMENTS_DOMAIN = "${readContent['ENTITLEMENTS_DOMAIN']}"
KEYCLOAK_CLIENT_ID = "${readContent['KEYCLOAK_CLIENT_ID']}"
KEYCLOAK_CLIENT_SECRET = "${readContent['KEYCLOAK_CLIENT_SECRET']}"
KEYCLOAK_REALM = "${readContent['KEYCLOAK_REALM']}"
KEYCLOAK_URL = "${readContent['KEYCLOAK_URL']}"
LEGAL_TAG = "${readContent['LEGAL_TAG']}"
OTHER_RELEVANT_DATA_COUNTRIES = "${readContent['OTHER_RELEVANT_DATA_COUNTRIES']}"
PROJECT_ID = "${readContent['PROJECT_ID']}"
SEARCH_HOST = "${readContent['SEARCH_HOST']}"
STORAGE_HOST = "${readContent['STORAGE_HOST']}"
USER_EMAIL = "${readContent['USER_EMAIL']}"
}
steps {
script {
sh 'mvn -f testing/integration-tests/search-test-ibm/pom.xml test'
}
}
}
}
}
\ No newline at end of file
......@@ -2,7 +2,7 @@
global:
# Service(s) Replica Count
replicaCount: 1
replicaCount: 3
################################################################################
# Specify the Gitlab branch being used for image creation
......@@ -12,3 +12,6 @@ image:
repository: #{container-registry}#.azurecr.io
branch: #{ENVIRONMENT_NAME}#
tag: #{Build.SourceVersion}#
policy:
enabled: #{POLICY_ENABLED}#
\ No newline at end of file
......@@ -41,9 +41,14 @@ spec:
imagePullPolicy: Always
ports:
- containerPort: 80
resources:
requests:
cpu: "100m"
limits:
cpu: "300m"
readinessProbe:
httpGet:
path: /api/search/v2/swagger-ui.html
path: /api/search/v2/actuator/health
port: 80
volumeMounts:
- name: azure-keyvault
......@@ -63,21 +68,6 @@ spec:
configMapKeyRef:
name: osdu-svc-properties
key: ENV_KEYVAULT
- name: AZURE_CLIENT_ID
valueFrom:
secretKeyRef:
name: active-directory
key: principal-clientid
- name: AZURE_CLIENT_SECRET
valueFrom:
secretKeyRef:
name: active-directory
key: principal-clientpassword
- name: AZURE_TENANT_ID
valueFrom:
secretKeyRef:
name: active-directory
key: tenantid
- name: aad_client_id
valueFrom:
secretKeyRef:
......@@ -88,12 +78,17 @@ spec:
secretKeyRef:
name: central-logging
key: appinsights
- name: REDIS_DATABASE
valueFrom:
configMapKeyRef:
name: redis-configmap
key: search-service
- name: APPLICATIONINSIGHTS_CONNECTION_STRING
value: InstrumentationKey=$(appinsights_key)
- name: cosmosdb_database
value: osdu-db
- name: entitlements_service_endpoint
value: http://entitlements-azure/entitlements/v1
value: http://entitlements/api/entitlements/v2
- name: entitlements_service_api_key
value: "OBSOLETE"
- name: ENVIRONMENT
......@@ -105,9 +100,13 @@ spec:
- name: MAX_CACHE_VALUE_SIZE
value: "60"
- name: search_service_spring_logging_level
value: debug
value: INFO
- name: partition_service_endpoint
value: http://partition/api/partition/v1
- name: policy_enabled
value: "{{ .Values.policy.enabled }}"
- name: policy_service_endpoint
value: http://policy-service/api/policy/v1
- name: azure_istioauth_enabled
value: "true"
- name: azure_activedirectory_AppIdUri
......
......@@ -13,9 +13,12 @@
# limitations under the License.
global:
replicaCount: 1
replicaCount: 2
image:
repository: community.opengroup.org:5555/osdu/platform/system/search-service
branch: master
tag: latest
policy:
enabled: true
\ No newline at end of file
......@@ -48,6 +48,8 @@ variables:
value: "devops/azure/chart/helm-config.yaml"
- name: 'MANIFEST_REPO'
value: $[ resources.repositories['FluxRepo'].name ]
- name: 'MAVEN_CACHE_FOLDER'
value: $(Pipeline.Workspace)/.m2/repository
- name: SKIP_TESTS
value: 'false'
......@@ -56,8 +58,8 @@ stages:
parameters:
mavenGoal: 'package'
mavenPublishJUnitResults: true
serviceCoreMavenOptions: '-P search-core'
mavenOptions: '-P search-azure'
serviceCoreMavenOptions: '-P search-core -Dmaven.repo.local=$(MAVEN_CACHE_FOLDER)'
mavenOptions: '-P search-azure -Dmaven.repo.local=$(MAVEN_CACHE_FOLDER)'
copyFileContents: |
pom.xml
provider/search-azure/maven/settings.xml
......
......@@ -48,6 +48,8 @@ variables:
value: "devops/azure/chart/helm-config.yaml"
- name: 'MANIFEST_REPO'
value: $[ resources.repositories['FluxRepo'].name ]
- name: 'MAVEN_CACHE_FOLDER'
value: $(Pipeline.Workspace)/.m2/repository
- name: SKIP_TESTS
value: 'false'
......@@ -56,8 +58,8 @@ stages:
parameters:
mavenGoal: 'package'
mavenPublishJUnitResults: true
serviceCoreMavenOptions: '-P search-core'
mavenOptions: '-P search-azure'
serviceCoreMavenOptions: '-P search-core -Dmaven.repo.local=$(MAVEN_CACHE_FOLDER)'
mavenOptions: '-P search-azure -Dmaven.repo.local=$(MAVEN_CACHE_FOLDER)'
copyFileContents: |
pom.xml
provider/search-azure/maven/settings.xml
......
# Patterns to ignore when building packages.
# This supports shell glob matching, relative path matching, and
# negation (prefixed with !). Only one pattern per line.
.DS_Store
# Common VCS dirs
.git/
.gitignore
.bzr/
.bzrignore
.hg/
.hgignore
.svn/
# Common backup files
*.swp
*.bak
*.tmp
*.orig
*~
# Various IDEs
.project
.idea/
*.tmproj
.vscode/
apiVersion: v2
name: gcp-search-configmap
description: A Helm chart for Kubernetes
# A chart can be either an 'application' or a 'library' chart.
#
# Application charts are a collection of templates that can be packaged into versioned archives
# to be deployed.
#
# Library charts provide useful utilities or functions for the chart developer. They're included as
# a dependency of application charts to inject those utilities and functions into the rendering
# pipeline. Library charts do not define any templates and therefore cannot be deployed.
type: application
# This is the chart version. This version number should be incremented each time you make changes
# to the chart and its templates, including the app version.
# Versions are expected to follow Semantic Versioning (https://semver.org/)
version: 0.1.0
# This is the version number of the application being deployed. This version number should be
# incremented each time you make changes to the application. Versions are not expected to
# follow Semantic Versioning. They should reflect the version the application is using.
# It is recommended to use it with quotes.
appVersion: "1.16.0"
{{/*
Expand the name of the chart.
*/}}
{{- define "configmap.name" -}}
{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }}
{{- end }}
{{/*
Create a default fully qualified app name.
We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec).
If release name contains chart name it will be used as a full name.
*/}}
{{- define "configmap.fullname" -}}
{{- if .Values.fullnameOverride }}
{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }}
{{- else }}
{{- $name := default .Chart.Name .Values.nameOverride }}
{{- if contains $name .Release.Name }}
{{- .Release.Name | trunc 63 | trimSuffix "-" }}
{{- else }}
{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }}
{{- end }}
{{- end }}
{{- end }}
{{/*
Create chart name and version as used by the chart label.
*/}}
{{- define "configmap.chart" -}}
{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }}
{{- end }}
{{/*
Common labels
*/}}
{{- define "configmap.labels" -}}
helm.sh/chart: {{ include "configmap.chart" . }}
{{ include "configmap.selectorLabels" . }}
{{- if .Chart.AppVersion }}
app.kubernetes.io/version: {{ .Chart.AppVersion | quote }}
{{- end }}
app.kubernetes.io/managed-by: {{ .Release.Service }}
{{- end }}
{{/*
Selector labels
*/}}
{{- define "configmap.selectorLabels" -}}
app.kubernetes.io/name: {{ include "configmap.name" . }}
app.kubernetes.io/instance: {{ .Release.Name }}
{{- end }}
{{/*
Create the name of the service account to use
*/}}
{{- define "configmap.serviceAccountName" -}}
{{- if .Values.serviceAccount.create }}
{{- default (include "configmap.fullname" .) .Values.serviceAccount.name }}
{{- else }}
{{- default "default" .Values.serviceAccount.name }}
{{- end }}
{{- end }}
apiVersion: v1
kind: ConfigMap
metadata:
labels:
app: {{ .Values.conf.app_name }}
name: {{ .Values.conf.configmap }}
namespace: {{ .Release.Namespace }}
data:
LOG_LEVEL: "{{ .Values.data.log_level }}"
ENVIRONMENT: {{ .Values.data.environment }}
GOOGLE_CLOUD_PROJECT: {{ .Values.data.google_cloud_project_id }}
REDIS_GROUP_HOST: "{{ .Values.data.redis_group_host }}"
REDIS_SEARCH_HOST: "{{ .Values.data.redis_search_host }}"
REDIS_SEARCH_PORT: "{{ .Values.data.redis_search_port }}"
INDEXER_HOST: {{ .Values.data.indexer_host }}
AUTHORIZE_API: {{ .Values.data.authorize_api }}
ENTITLEMENTS_HOST: {{ .Values.data.authorize_api }}
SECURITY_HTTPS_CERTIFICATE_TRUST: "{{ .Values.data.security_https_certificate_trust }}"
GOOGLE_AUDIENCES: "{{ .Values.data.audiences }}"
PARTITION_API: "{{ .Values.data.partition_api }}"
POLICY_API: "{{ .Values.data.policy_api }}"
# Default values for configmap.
# This is a YAML-formatted file.
# Declare variables to be passed into your templates.
data:
log_level: "INFO"
environment: "dev"
google_cloud_project_id: ""
redis_group_host: ""
redis_search_host: ""
redis_search_port: "6379"
indexer_host: ""
authorize_api: ""
security_https_certificate_trust: "true"
audiences: ""
partition_api: ""
policy_api: ""
conf:
configmap: "search-config"
app_name: "search"
# Patterns to ignore when building packages.
# This supports shell glob matching, relative path matching, and
# negation (prefixed with !). Only one pattern per line.
.DS_Store
# Common VCS dirs
.git/
.gitignore
.bzr/
.bzrignore
.hg/
.hgignore
.svn/
# Common backup files
*.swp
*.bak
*.tmp
*.orig
*~
# Various IDEs
.project
.idea/
*.tmproj
.vscode/
apiVersion: v2
name: gcp-search-deploy
description: A Helm chart for Kubernetes
# A chart can be either an 'application' or a 'library' chart.
#
# Application charts are a collection of templates that can be packaged into versioned archives
# to be deployed.
#
# Library charts provide useful utilities or functions for the chart developer. They're included as
# a dependency of application charts to inject those utilities and functions into the rendering
# pipeline. Library charts do not define any templates and therefore cannot be deployed.
type: application
# This is the chart version. This version number should be incremented each time you make changes
# to the chart and its templates, including the app version.
# Versions are expected to follow Semantic Versioning (https://semver.org/)
version: 0.1.0
# This is the version number of the application being deployed. This version number should be
# incremented each time you make changes to the application. Versions are not expected to
# follow Semantic Versioning. They should reflect the version the application is using.
# It is recommended to use it with quotes.
appVersion: "1.16.0"
{{/*
Expand the name of the chart.
*/}}
{{- define "deploy.name" -}}
{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }}
{{- end }}
{{/*
Create a default fully qualified app name.
We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec).
If release name contains chart name it will be used as a full name.
*/}}
{{- define "deploy.fullname" -}}
{{- if .Values.fullnameOverride }}
{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }}
{{- else }}
{{- $name := default .Chart.Name .Values.nameOverride }}
{{- if contains $name .Release.Name }}
{{- .Release.Name | trunc 63 | trimSuffix "-" }}
{{- else }}
{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }}
{{- end }}
{{- end }}
{{- end }}
{{/*
Create chart name and version as used by the chart label.
*/}}
{{- define "deploy.chart" -}}
{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }}
{{- end }}
{{/*
Common labels
*/}}
{{- define "deploy.labels" -}}
helm.sh/chart: {{ include "deploy.chart" . }}
{{ include "deploy.selectorLabels" . }}
{{- if .Chart.AppVersion }}
app.kubernetes.io/version: {{ .Chart.AppVersion | quote }}
{{- end }}
app.kubernetes.io/managed-by: {{ .Release.Service }}
{{- end }}
{{/*
Selector labels
*/}}
{{- define "deploy.selectorLabels" -}}
app.kubernetes.io/name: {{ include "deploy.name" . }}
app.kubernetes.io/instance: {{ .Release.Name }}
{{- end }}
{{/*
Create the name of the service account to use
*/}}
{{- define "deploy.serviceAccountName" -}}
{{- if .Values.serviceAccount.create }}
{{- default (include "deploy.fullname" .) .Values.serviceAccount.name }}
{{- else }}
{{- default "default" .Values.serviceAccount.name }}
{{- end }}
{{- end }}
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment