diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 421acb135e0adb326f7ca4a7ee69d3bd2a3f950b..22439edb399d69e298f282ee00612cfced102a68 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -61,12 +61,14 @@ unit_tests: - tests/./unit_tests.sh || EXIT_CODE=$? - exit ${EXIT_CODE} -dags_rsync_community: +dags_rsync: stage: deploy script: - gcloud auth activate-service-account --key-file $OSDU_GCP_DEPLOY_FILE - cd src - - gsutil -m rsync -x "\.git.*|tests/.*|plugins/tests.*$" -r "$PWD" $OSDU_GCP_DEPL_TARGET + - gsutil -m rsync -x "dags/libs*" -d -R dags $OSDU_GCP_DEPL_TARGET/dags/ingestion + - gsutil -m rsync -d -R dags/libs $OSDU_GCP_DEPL_TARGET/dags/libs + - gsutil -m rsync -R plugins $OSDU_GCP_DEPL_TARGET/plugins only: variables: - $CI_COMMIT_REF_PROTECTED diff --git a/src/dags/default-ingest.py b/src/dags/default-ingest.py index 4ea1a84ac411ace3754f995a904bad852e8c11ef..0d0758d1b72c89182abb07a4b6228baf36101f42 100644 --- a/src/dags/default-ingest.py +++ b/src/dags/default-ingest.py @@ -23,7 +23,7 @@ from airflow import DAG from airflow.operators import UpdateStatusOperator from airflow.operators.python_operator import PythonOperator -from create_records import create_records #isort:skip +from libs.create_records import create_records #isort:skip """ A workflow creating a record diff --git a/src/dags/create_records.py b/src/dags/libs/create_records.py similarity index 100% rename from src/dags/create_records.py rename to src/dags/libs/create_records.py diff --git a/src/dags/osdu-ingest.py b/src/dags/osdu-ingest.py index 9fdfc5311c7357e87c42a929a41c299ff7e84358..9c602285728f71d88d49bc971c80a0f110b01c0f 100644 --- a/src/dags/osdu-ingest.py +++ b/src/dags/osdu-ingest.py @@ -61,4 +61,5 @@ search_record_ids_op = SearchRecordIdOperator( dag=dag ) -update_status_running_op >> process_manifest_op >> search_record_ids_op >> update_status_finished_op +update_status_running_op >> process_manifest_op >> \ +search_record_ids_op >> update_status_finished_op # pylint: disable=pointless-statement diff --git a/src/plugins/__init__.py b/src/plugins/__init__.py index 59ea0498909ef9cd595895d7046ef06b37d9d2ab..d6cb30bc70e4adcb525269d15429e6249c5f0daa 100644 --- a/src/plugins/__init__.py +++ b/src/plugins/__init__.py @@ -1,6 +1,26 @@ +# Copyright 2020 Google LLC +# Copyright 2020 EPAM Systems +# Copyright 2020 Amazon +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +"""isort:skip_file""" from airflow.plugins_manager import AirflowPlugin -from .operators import ProcessManifestOperator, SearchRecordIdOperator, UpdateStatusOperator +from operators.search_record_id_op import SearchRecordIdOperator # pylint: disable=import-error + +from .operators import ProcessManifestOperator, UpdateStatusOperator # Defining the plugin class