Commit 33a10ee9 authored by Elsi Abraham's avatar Elsi Abraham
Browse files

Merge branch 'master' into aws-update-eks-fix

parents 85985dc9 c5b46c4f
Pipeline #116762 failed with stages
in 86 minutes and 15 seconds
# Copyright 2021 Schlumberger
# Copyright 2021 EPAM
# Copyright 2022 EPAM
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
......@@ -45,7 +45,7 @@ include:
file: "scanners/gitlab-ultimate.yml"
- project: "osdu/platform/ci-cd-pipelines"
file: "cloud-providers/osdu-gcp-gke.yml"
file: "cloud-providers/osdu-gcp-global.yml"
- project: "osdu/platform/ci-cd-pipelines"
file: "cloud-providers/ibm-wellbore-git.yml"
......@@ -135,7 +135,6 @@ containerize:
- echo ---- ---- PUSH IMAGE
- docker push $IMAGE_TAG
- docker push $CI_REGISTRY_IMAGE:latest
# Allow failure on private development deployments
ibm-deploy-devpri:
allow_failure: true
......
......@@ -179,7 +179,7 @@ class ConfigurationContainer:
modules: EnvVar = EnvVar(
key='MODULES',
description="""Comma separated list of module names to load.""",
default="log_recognition.routers.log_recognition") # Add modules to the list once they are refactored, so that they are included
default="")
min_worker_memory: EnvVar = EnvVar(
key='MIN_WORKER_MEMORY',
......
......@@ -21,9 +21,8 @@ from typing import Optional, List
from pydantic import BaseModel, Field
import odes_storage.models as model
import app.modules.log_recognition.routers.family_processor_manager as fp_manager
from . import family_processor_manager as fp_manager
from app.clients.storage_service_client import get_storage_record_service
from app.conf import Config
from app.routers.common_parameters import REQUIRED_ROLES_WRITE
from app.context import Context, get_ctx
from app.helper.traces import with_trace
......@@ -33,6 +32,7 @@ from app.helper.traces import TracingRoute
router = APIRouter(route_class=TracingRoute)
router.prefix = '/log-recognition'
router.tags = ['log-recognition']
CUSTOM_CATALOG_LIFETIME = 300 # in seconds
class CatalogItem(BaseModel):
......@@ -96,7 +96,7 @@ class CatalogRecord(BaseModel):
}
family_processor_manager = fp_manager.FamilyProcessorManager(Config.custom_catalog_timeout.value)
family_processor_manager = fp_manager.FamilyProcessorManager(CUSTOM_CATALOG_LIFETIME)
class GuessRequest(BaseModel):
......
......@@ -25,7 +25,6 @@ from app.conf import Config, check_environment
from app.errors.exception_handlers import add_exception_handlers, create_custom_http_exception_handler
from app.helper.traces import TracingRoute
from app.model.entity_utils import Entity
from app.modules import discoverer
from app.helper import traces, logger, metric
from app.injector.app_injector import AppInjector
......@@ -51,6 +50,7 @@ from app.routers.ddms_v3 import (
from app.routers.bulk import bulk_routes, statistics_routes
from app.routers.trajectory import trajectory_ddms_v2
from app.routers.dipset import dipset_ddms_v2, dip_ddms_v2
from app.routers.log_recognition import log_recognition
from app.routers.search import search, fast_search, search_v3, fast_search_v3, search_v3_alpha
from app.clients import StorageRecordServiceClient, SearchServiceClient
from app.pool_executor import run_in_pool_executor
......@@ -157,7 +157,6 @@ async def startup_event():
for _ in range(POOL_EXECUTOR_MAX_WORKER):
asyncio.create_task(run_in_pool_executor(executor_startup_task))
add_modules_routers()
metric.init_metric(wdms_app)
......@@ -299,6 +298,16 @@ wdms_app.include_router(
],
include_in_schema=True)
# ---------------------------------------------------------------------------------------------------------------------
# ---------------------------------------------------------------------------------------------------------------------
# ------------------------------------------- Log recognition ---------------------------------------------------------
# ---------------------------------------------------------------------------------------------------------------------
# ---------------------------------------------------------------------------------------------------------------------
wdms_app.include_router(log_recognition.router,
dependencies=[Depends(require_data_partition_id, use_cache=False),
Depends(require_opendes_authorized_user, use_cache=False)])
# ---------------------------------------------------------------------------------------------------------------------
# ---------------------------------------------------------------------------------------------------------------------
# ---------------------------------------- Deprecated API set ---------------------------------------------------------
......@@ -364,27 +373,3 @@ wdms_app.add_middleware(CreateBasicContextMiddleware, config=Config, injector=ap
# adding exception handling
add_exception_handlers(wdms_app)
def remove_modules_routers():
discoverer.reset_routers()
# Load and add router modules
def add_modules_routers():
for router in discoverer.get_routers():
add_modules_router(router)
def add_modules_router(router):
log = logger.get_logger()
name = router.prefix
try:
log.info(f'Adding router family `{name}`')
wdms_app.include_router(router, dependencies=[Depends(require_data_partition_id, use_cache=False),
Depends(require_opendes_authorized_user, use_cache=False)])
log.info(f'Done. `{name}` added')
except ValueError as error:
log.warning(f'Failed to add `{name}` router. {error}')
except:
log.warning(f'Failed to add `{name}` router. {sys.exc_info()[0]}')
......@@ -80,7 +80,6 @@ configMap:
usePartitionService: 'enabled'
loggerLevel: 'INFO'
environmentName: gitlab
modules: log_recognition.routers.log_recognition
name: os-wellbore-ddms-envs
ingress:
......
......@@ -8,7 +8,7 @@ metadata:
data:
{{- if not .Values.conf.onPremEnabled }}
CLOUD_PROVIDER: "gcp"
OS_WELLBORE_DDMS_DATA_PROJECT_ID: "{{ .Values.data.osWellboreDdmsDataProjectId }}"
OS_WELLBORE_DDMS_DATA_PROJECT_ID: "{{ .Values.data.projectId }}"
{{- else }}
KEYCLOAK_AUTH: "yes"
BASE_BUCKET: "wellbore"
......
......@@ -3,7 +3,7 @@
# Declare variables to be passed into your templates.
data:
osWellboreDdmsDataProjectId: ""
projectId: ""
conf:
configmap: "wellbore-config"
......
......@@ -7,7 +7,7 @@ data:
requestsMemory: "1G"
limitsCpu: "1"
limitsMemory: "3G"
image: "community.opengroup.org:5555/osdu/platform/domain-data-mgmt-services/wellbore/wellbore-domain-services/osdu-gcp-wellbore:latest"
image: ""
imagePullPolicy: "IfNotPresent"
serviceAccountName: "wellbore"
......
variables:
OSDU_GCP_SERVICE: wellbore
OSDU_GCP_VENDOR: gcp
OSDU_GCP_HELM_PACKAGE_CHARTS: "devops/gcp/deploy devops/gcp/configmap"
OSDU_GCP_HELM_CONFIG_SERVICE_VARS: "--set data.osWellboreDdmsDataProjectId=$OSDU_GCP_PROJECT"
OSDU_GCP_HELM_CONFIG_SERVICE_VARS_DEV2: "--set data.osWellboreDdmsDataProjectId=$OSDU_GCP_PROJECT"
OSDU_GCP_HELM_CONFIG_SERVICE_VARS: "--set data.projectId=$OSDU_GCP_PROJECT"
OSDU_GCP_HELM_CONFIG_SERVICE_VARS_DEV2: "--set data.projectId=$OSDU_GCP_PROJECT"
OSDU_GCP_HELM_DEPLOYMENT_SERVICE_VARS: >
--set data.image=$CI_REGISTRY_IMAGE/osdu-gcp-$OSDU_GCP_SERVICE:$CI_COMMIT_SHORT_SHA
--set data.serviceAccountName=$OSDU_GCP_SERVICE-k8s
......@@ -21,11 +20,11 @@ osdu-gcp-containerize-gitlab:
--build-arg commit_id=$CI_COMMIT_SHORT_SHA
--build-arg build_origin="Gitlab"
--build-arg commit_branch=$CI_COMMIT_BRANCH
# should be deleted after release 0.15
# should be deleted after release 0.15
before_script:
- sed -i "s|osdu-api==0.14.0|osdu_api~=0.15.0.dev|g" requirements.txt
- sed -i "s|osdu-core-lib-python-anthos==1.0.0|osdu-core-lib-python-anthos==1.0.1|g" requirements.txt
- sed -i "s|osdu-core-lib-python-aws==1.0.1|# osdu-core-lib-python-aws==1.0.1|g" requirements.txt
- sed -i "s|osdu-api==0.14.0|osdu_api~=0.15.0.dev|g" requirements.txt
- sed -i "s|osdu-core-lib-python-anthos==1.0.0|osdu-core-lib-python-anthos==1.0.1|g" requirements.txt
- sed -i "s|osdu-core-lib-python-aws==1.0.1|# osdu-core-lib-python-aws==1.0.1|g" requirements.txt
osdu-gcp-containerize-gcr:
variables:
......@@ -139,7 +138,7 @@ osdu-gcp-anthos-test-python:
--legal_tag $LEGAL_TAG
- pytest ./functional --environment="./generated/postman_environment.json" --filter-tag='!search|!chunking'
# Disable maven job in gcp common gke pipeline
# Disable maven job in gcp common gke pipelines
osdu-gcp-test:
extends:
- .osdu-gcp-variables
......
import pytest
from wdms_client.request_builders import make_base_request_proto
from wdms_client.request_runner import RequestRunner
from .fixtures import with_wdms_env
@pytest.mark.tag('smoke', 'log_recognition')
def test_basic_log_recognition(with_wdms_env):
result = RequestRunner(make_base_request_proto('POST', '/log-recognition/family', payload={
"label": "GRD",
"log_unit": "GAPI",
"description": "LDTD Gamma Ray"
})).call(with_wdms_env)
result.assert_ok()
resobj = result.get_response_obj()
assert resobj["family"] == "Gamma Ray"
assert resobj["log_unit"] == "GAPI"
assert resobj["base_unit"] == "gAPI"
......@@ -21,6 +21,7 @@ from .wdms import error_cases
from .wdms import model_extensibility
from .wdms import recursive_delete
from .wdms import search_apis
from ..request_runner import Request
from deepdiff import DeepDiff
request_path_dict = {
......@@ -268,3 +269,23 @@ def diff_records(ref, res):
def diff_record_against_ref(kind: str, res_dict: dict):
ref = get_cleaned_ref_and_res(kind)
return diff_records(ref, res_dict)
def make_base_request_proto(method: str, path: str, name=None, payload=None, content_type='application/json'):
import json
if payload is not None and not isinstance(payload, str) and content_type == 'application/json':
payload = json.dumps(payload)
return Request(
name=name or f"{method} - {path}",
method=method,
url='{{base_url}}' + path,
headers={
'accept': 'application/json',
'data-partition-id': '{{data_partition}}',
'Connection': '{{header_connection}}',
'Authorization': 'Bearer {{token}}',
'Content-Type': content_type
},
payload=payload
)
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment