Merge branch 'feature/unified_cicd' into 'master'

GONRG-474 Unified CICD and folders structure

See merge request !4

.gitlab-ci.yml

@@ -61,12 +61,14 @@ unit_tests:
     - tests/./unit_tests.sh  || EXIT_CODE=$?
     - exit ${EXIT_CODE}
 
-dags_rsync_community:
+dags_rsync:
   stage: deploy
   script:
     - gcloud auth activate-service-account --key-file $OSDU_GCP_DEPLOY_FILE
     - cd src
-    - gsutil -m rsync -x "\.git.*|tests/.*|plugins/tests.*$" -r "$PWD" $OSDU_GCP_DEPL_TARGET
+    - gsutil -m rsync -x "dags/libs*" -d -R dags  $OSDU_GCP_DEPL_TARGET/dags/ingestion
+    - gsutil -m rsync -d -R dags/libs  $OSDU_GCP_DEPL_TARGET/dags/libs
+    - gsutil -m rsync -R plugins $OSDU_GCP_DEPL_TARGET/plugins
   only:
     variables:
         - $CI_COMMIT_REF_PROTECTED

src/dags/default-ingest.py

@@ -23,7 +23,7 @@ from airflow import DAG
 from airflow.operators import UpdateStatusOperator
 from airflow.operators.python_operator import PythonOperator
 
-from create_records import create_records #isort:skip
+from libs.create_records import create_records #isort:skip
 
 """
 A workflow creating a record

src/dags/osdu-ingest.py

@@ -61,4 +61,5 @@ search_record_ids_op = SearchRecordIdOperator(
     dag=dag
 )
 
-update_status_running_op >> process_manifest_op >> search_record_ids_op >> update_status_finished_op
+update_status_running_op >> process_manifest_op >> \
+search_record_ids_op >> update_status_finished_op # pylint: disable=pointless-statement

src/plugins/__init__.py

+#  Copyright 2020 Google LLC
+#  Copyright 2020 EPAM Systems
+#  Copyright 2020 Amazon
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+
+"""isort:skip_file"""
 from airflow.plugins_manager import AirflowPlugin
 
-from .operators import ProcessManifestOperator, SearchRecordIdOperator, UpdateStatusOperator
+from operators.search_record_id_op import SearchRecordIdOperator  # pylint: disable=import-error
+
+from .operators import ProcessManifestOperator, UpdateStatusOperator
 
 
 # Defining the plugin class