Commit 5ad788c3 authored by Luc Yriarte's avatar Luc Yriarte
Browse files

Merge branch 'extensions-as-modules' into 'master'

Remove prototyping (extensibility)

See merge request !165
parents 90a728d3 286f2c5b
Pipeline #51842 passed with stages
in 12 minutes and 4 seconds
......@@ -183,10 +183,10 @@ class ConfigurationContainer:
default='300',
factory=lambda x: int(x))
extension_modules: EnvVar = EnvVar(
key='EXTENSION_MODULES',
description="""Comma separated list of extension module names to load.""",
default=None)
modules: EnvVar = EnvVar(
key='MODULES',
description="""Comma separated list of module names to load.""",
default="") # Add modules to the list once they are refactored, so that they are included
_environment_dict: Dict = os.environ
......
# Wellbore Domain Services Extensions
> :warning: **This is an alpha feature**
> Implementation changes are expected, and will not be announced.
[[_TOC_]]
## Use cases
### Include new routers
1. Add a new directory `<extension package name>/routers` under `app/extensions`
2. Add a new python module `<router module name>.py`
```
/wellbore-domain-services
- app
|- extensions
|- <extension package name>
|- __init__.py
|- routers
|- <router module name>.py
|- __init__.py
```
3. In `<router module name>.py` include:
```python
from fastapi import APIRouter
router = APIRouter()
router.prefix = '<extension router prefix>'
router.tags = ['<extension router tags>']
def can_run() -> (bool, str):
# Include the extension router specific checks if applicable.
# WDMS main app will skip loading the extension module routers if it returns False.
return True
```
4. Include new service endpoints to `router` in `<router module name>.py`.
- Use `@router.[get|post|delete|put|patch|...]` fastapi decorator to include new methods to the router
- Check WDMS routers implementation under `app/routers`
- The same pattern is encouraged to be used in the extension routers.
```python
from fastapi import APIRouter, Depends, status
from app.utils import Context, get_ctx
from app.model.model_curated import *
from app.clients.storage_service_client import get_storage_record_service
from app.model.model_utils import from_record
router = APIRouter() # From previous step
# E.g.: Including a GET method API
@router.get('/markers/{marker_id}',
response_model=marker,
summary="Get the marker using wks:marker:1.0.4 schema",
description="""Get the Marker object using its **id**.""",
operation_id="get_marker",
responses={status.HTTP_404_NOT_FOUND: {"description": "marker not found"}},
response_model_exclude_unset=True)
async def get_marker(
marker_id: str,
ctx: Context = Depends(get_ctx)
) -> marker:
storage_client = await get_storage_record_service(ctx)
marker_record = await storage_client.get_record(id=marker_id, data_partition_id=ctx.partition_id)
return from_record(marker, marker_record)
```
5. Include the full router module name in `EXTENSION_MODULES` environment variable
E.g.: `app.extensions.<extension package name>.routers.<router module name>`
6. Update the deployment scripts accordingly.
7. Run WDMS service, and check in the logs for the messages
```
Loading `app.extensions.<extension package>.routers.<router module>` extension
Done. `app.extensions.<extension package>.routers.<router module>` loaded
```
8. Verify the new API endpoints were added under the provided tag,
and with the given prefix. `https://{base_url}/api/os-wellbore-ddms/docs/{Tag}`
9. Include test
- Unit tests are to be placed under `tests/unit/extensions/<extension package name>`
- Integration tests are to be placed under `tests/integration/functional/extensions/<extension package name>`
#### Troubleshooting
##### A. Wrong router module configuration
Check step 3 above
```shell
Loading `app.extensions.{}.routers.{}` extension
Failed to load `app.extensions.{}.routers.{}` extension.
Module not configured properly. module 'app.extensions.{}.routers.{}' has no attribute 'router'
```
##### B. Wrong module name
Review steps 2 and 4 above
```shell
Loading `app.extensions.{}.routers.{}` extension
Failed to load `app.extensions.{}.routers.{}` extension.
Module not found.
No module named 'app.extensions.{}.routers.{}'
```
##### C. Trailing comma in `EXTENSION_MODULES` list
Review step 4 above, make sure there is no trailing comma in the `EXTENSION_MODULES` list.
```shell
Loading `` extension
Failed to load `` extension. Empty module name
```
##### D. Empty router prefix or tags
Check step 3 above
```shell
Loading `app.extensions.{}.routers.{}` extension
Failed to load `app.extensions.{}.routers.{}` extension.
Module not configured properly. Router prefix cannot be empty.
```
\ No newline at end of file
# Wellbore Domain Services Modules
> :warning: **Work In Progress**
> Refactoring of Wellbore DDMS as independent modules
>
\ No newline at end of file
# Load extension modules [alpha version]
# Load modules [alpha version]
import importlib
import sys
......@@ -12,26 +12,26 @@ discovered_routers = []
def get_routers():
if len(discovered_routers) == 0:
load_extensions()
load_modules()
return discovered_routers
def load_extensions():
discovered_extensions = []
def load_modules():
discovered_modules = []
extension_modules = Config.extension_modules.value
if extension_modules:
discovered_extensions = extension_modules.split(',')
modules = Config.modules.value
if modules:
discovered_modules = modules.split(',')
logger.get_logger().info(f'Discovered extensions: {discovered_extensions}')
for name in discovered_extensions:
load_extension(name)
logger.get_logger().info(f'Discovered modules: {discovered_modules}')
for name in discovered_modules:
load_module(name)
def load_extension(name):
def load_module(name):
log = logger.get_logger()
try:
log.info(f'Loading `{name}` extension')
log.info(f'Loading `{name}` module')
module = importlib.import_module(name)
can_run, message = module.can_run()
......@@ -49,12 +49,12 @@ def load_extension(name):
log.info(f'Done. `{name}` loaded')
except AttributeError as error:
log.warning(f'Failed to load `{name}` extension. Module not configured properly. {error}')
log.warning(f'Failed to load `{name}` module. Module not configured properly. {error}')
except ModuleNotFoundError as error:
log.warning(f'Failed to load `{name}` extension. Module not found. {error}')
log.warning(f'Failed to load `{name}` module. Module not found. {error}')
except ValueError as error:
log.warning(f'Failed to load `{name}` extension. {error}')
log.warning(f'Failed to load `{name}` module. {error}')
except NameError as error:
log.warning(f'Failed to load `{name}` extension. Missing module configuration. {error}')
log.warning(f'Failed to load `{name}` module. Missing module configuration. {error}')
except:
log.warning(f'Failed to load `{name}` extension. {sys.exc_info()[0]}')
log.warning(f'Failed to load `{name}` module. {sys.exc_info()[0]}')
......@@ -23,7 +23,7 @@ from app import __version__, __build_number__, __app_name__
from app.auth.auth import require_opendes_authorized_user
from app.conf import Config, check_environment
from app.errors.exception_handlers import add_exception_handlers
from app.extensions import discoverer
from app.modules import discoverer
from app.helper import traces, logger
from app.injector.app_injector import AppInjector
......@@ -125,7 +125,7 @@ async def startup_event():
if Config.alpha_feature_enabled.value:
enable_alpha_feature()
add_extension_routers()
add_modules_routers()
@base_app.on_event('shutdown')
......@@ -279,13 +279,13 @@ wdms_app.add_middleware(CreateBasicContextMiddleware, injector=app_injector)
add_exception_handlers(wdms_app)
# Load and add router extensions [alpha version]
def add_extension_routers():
# Load and add router modules [alpha version]
def add_modules_routers():
for router in discoverer.get_routers():
add_extension_router(router)
add_modules_router(router)
def add_extension_router(router):
def add_modules_router(router):
log = logger.get_logger()
name = router.prefix
try:
......
......@@ -23,8 +23,8 @@ data:
USE_PARTITION_SERVICE: {{ .Values.configMap.data.usePartitionService }}
AZ_LOGGER_LEVEL: {{ .Values.configMap.data.loggerLevel }}
ENVIRONMENT_NAME: {{ .Values.configMap.data.environmentName }}
{{ if .Values.configMap.data.extensionModules }}
EXTENSION_MODULES: {{ .Values.configMap.data.extensionModules }}
{{ if .Values.configMap.data.modules }}
MODULES: {{ .Values.configMap.data.modules }}
{{ end }}
kind: ConfigMap
metadata:
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment