Skip to content
Snippets Groups Projects
Commit 566aae73 authored by MANISH KUMAR's avatar MANISH KUMAR
Browse files

Merge branch 'master' into 'users/nikhil/updateCoreCommon' 

# Conflicts:
#   NOTICE
parents 730518ab a50b4038
No related branches found
No related tags found
1 merge request!123update core common - Register
Pipeline #72801 failed
Stage: .pre
Stage: review
Stage: build
Pipeline: Register

#72802

    Hide whitespace changes
    Inline Side-by-side
    Showing
    with 70 additions and 44 deletions
    .gitlab-ci.yml
    + 4
    − 10
    View file @ 566aae73
    variables:
    OSDU_GCP_APPLICATION_NAME: os-register
    OSDU_GCP_VENDOR: gcp
    OSDU_GCP_ENVIRONMENT: dev
    OSDU_GCP_SERVICE: register
    OSDU_GCP_ENV_VARS: GOOGLE_CLOUD_PROJECT=${OSDU_GCP_PROJECT},ENTITLEMENTS_API=${OSDU_GCP_ENTITLEMENTS_V2_URL},PARTITION_API=${OSDU_GCP_PARTITION_API},GOOGLE_AUDIENCES=${GOOGLE_AUDIENCE},GCLOUD_REGION=${OSDU_GCP_CLOUDRUN_REGION},STORAGE_API=${OSDU_GCP_STORAGE_URL},INTEGRATION_TEST_AUDIENCES=${GOOGLE_AUDIENCE},SUBSCRIBER_SECRET=${OSDU_GCP_SUBSCRIBER_SECRET},SUBSCRIBER_PRIVATE_KEY_ID=${OSDU_GCP_SUBSCRIBER_PRIVATE_KEY_ID},ENVIRONMENT=${OSDU_GCP_ENVIRONMENT},CRON_JOB_EXPECTED_IP=${CRON_JOB_EXPECTED_IP},RECORDS_CHANGE_PUBSUB_ENDPOINT=${RECORDS_CHANGE_PUBSUB_ENDPOINT},SERVICE_IDENTITY=${SERVICE_IDENTITY}
    OSDU_GCP_TEST_SUBDIR: testing/$OSDU_GCP_SERVICE-test-$OSDU_GCP_VENDOR
    OSDU_GCP_LOG_LEVEL: INFO
    OSDU_GCP_VENDOR: gcp
    OSDU_GCP_HELM_CONFIG_SERVICE: register-config
    OSDU_GCP_HELM_DEPLOYMENT_SERVICE: register-deploy
    OSDU_GCP_HELM_CONFIG_SERVICE_VARS: "--set data.subscriber_private_key_id=$OSDU_GCP_SUBSCRIBER_PRIVATE_KEY_ID --set data.google_cloud_project=$OSDU_GCP_PROJECT --set data.google_audiences=$GOOGLE_AUDIENCE --set data.gcloud_region=$OSDU_GCP_REGION --set data.environment=$ENVIRONMENT --set data.records_change_pubsub_endpoint=$RECORDS_CHANGE_PUBSUB_ENDPOINT --set data.service_identity=$SERVICE_IDENTITY --set data.log_level=$OSDU_GCP_LOG_LEVEL"
    OSDU_GCP_HELM_DEPLOYMENT_SERVICE_VARS: "--set data.image=$CI_REGISTRY_IMAGE/osdu-gcp:$CI_COMMIT_SHORT_SHA --set data.serviceAccountName=workload-identity-register"
    OSDU_GCP_HELM_PACKAGE_CHARTS: "devops/gcp/deploy devops/gcp/configmap"
    OSDU_GCP_HELM_CONFIG_SERVICE_VARS: "--set data.google_cloud_project=$OSDU_GCP_PROJECT --set data.google_audiences=$GOOGLE_AUDIENCE --set data.gcloud_region=$OSDU_GCP_CLOUDRUN_REGION --set data.environment=$OSDU_GCP_ENVIRONMENT --set data.records_change_pubsub_endpoint=$RECORDS_CHANGE_PUBSUB_ENDPOINT --set data.service_identity=$SERVICE_IDENTITY"
    OSDU_GCP_HELM_DEPLOYMENT_SERVICE_VARS: "--set data.image=$CI_REGISTRY_IMAGE/osdu-gcp:$CI_COMMIT_SHORT_SHA --set data.serviceAccountName=workload-identity-register --set data.limits_cpu=1 --set data.limits_memory=1G"
    AWS_BUILD_SUBDIR: provider/register-aws/build-aws
    AWS_TEST_SUBDIR: testing/register-test-aws
    ......@@ -50,7 +44,7 @@ include:
    file: "scanners/gitlab-ultimate.yml"
    - project: "osdu/platform/ci-cd-pipelines"
    file: "cloud-providers/osdu-gcp-cloudrun.yml"
    file: "cloud-providers/osdu-gcp-gke.yml"
    - project: "osdu/platform/ci-cd-pipelines"
    ref: master
    ......
    NOTICE
    + 4
    − 2
    View file @ 566aae73
    ......@@ -104,6 +104,7 @@ The following software have components provided under the terms of this license:
    - Hibernate Validator Engine (from https://repo1.maven.org/maven2/org/hibernate/hibernate-validator)
    - Hibernate Validator Engine (from https://repo1.maven.org/maven2/org/hibernate/validator/hibernate-validator)
    - Hibernate Validator Engine (from https://repo1.maven.org/maven2/org/hibernate/validator/hibernate-validator)
    - Hibernate Validator Engine (from https://repo1.maven.org/maven2/org/hibernate/hibernate-validator)
    - IBM COS Java SDK for Amazon S3 (from https://github.com/ibm/ibm-cos-sdk-java)
    - IBM COS Java SDK for COS KMS (from https://github.com/ibm/ibm-cos-sdk-java)
    - IBM COS SDK For Java (from https://github.com/ibm/ibm-cos-sdk-java)
    ......@@ -174,10 +175,9 @@ The following software have components provided under the terms of this license:
    - Lucene Highlighter (from https://repo1.maven.org/maven2/org/apache/lucene/lucene-highlighter)
    - Lucene Join (from https://repo1.maven.org/maven2/org/apache/lucene/lucene-join)
    - Lucene Join (from https://repo1.maven.org/maven2/org/apache/lucene/lucene-join)
    - Lucene Memory (from https://repo1.maven.org/maven2/org/apache/lucene/lucene-memory)
    - Lucene Memory (from https://repo1.maven.org/maven2/org/apache/lucene/lucene-backward-codecs)
    - Lucene Memory (from https://repo1.maven.org/maven2/org/apache/lucene/lucene-memory)
    - Lucene Memory (from https://repo1.maven.org/maven2/org/apache/lucene/lucene-backward-codecs)
    - Lucene Memory (from https://repo1.maven.org/maven2/org/apache/lucene/lucene-memory)
    - Lucene Miscellaneous (from https://repo1.maven.org/maven2/org/apache/lucene/lucene-misc)
    - Lucene Miscellaneous (from https://repo1.maven.org/maven2/org/apache/lucene/lucene-misc)
    - Lucene Queries (from https://repo1.maven.org/maven2/org/apache/lucene/lucene-queries)
    ......@@ -400,8 +400,10 @@ The following software have components provided under the terms of this license:
    - rank-eval (from https://github.com/elastic/elasticsearch)
    - resilience4j (from https://resilience4j.readme.io)
    - resilience4j (from https://github.com/resilience4j/resilience4j)
    - resilience4j (from https://resilience4j.readme.io)
    - resilience4j (from https://github.com/resilience4j/resilience4j)
    - resilience4j (from https://resilience4j.readme.io)
    - resilience4j (from https://resilience4j.readme.io)
    - resilience4j (from https://github.com/resilience4j/resilience4j)
    - resilience4j (from https://resilience4j.readme.io)
    - resilience4j (from https://github.com/resilience4j/resilience4j)
    ......
    devops/gcp/configmap/templates/register-variables.yml
    + 1
    − 0
    View file @ 566aae73
    ......@@ -16,3 +16,4 @@ data:
    SERVICE_IDENTITY: "{{ .Values.data.service_identity }}"
    KEY_RING: "{{ .Values.data.key_ring }}"
    KMS_KEY: "{{ .Values.data.kms_key }}"
    SUBSCRIBER_PRIVATE_KEY_ID: "{{ .Values.data.subscriber_private_key_id }}"
    devops/gcp/configmap/values.yaml
    + 2
    − 2
    View file @ 566aae73
    # Default values for indexer-queue-configmap.
    # Default values for register-configmap.
    # This is a YAML-formatted file.
    # Declare variables to be passed into your templates.
    ......@@ -13,7 +13,7 @@ data:
    service_identity: ""
    key_ring: "csqp"
    kms_key: "registerService"
    subscriber_private_key_id: "testkey"
    conf:
    configmap: "register-config"
    ......
    devops/gcp/deploy/values.yaml
    + 1
    − 1
    View file @ 566aae73
    # Default values for indexer-queue-deploy.
    # Default values for register-deploy.
    # This is a YAML-formatted file.
    # Declare variables to be passed into your templates.
    ......
    provider/register-gcp/src/main/java/org/opengroup/osdu/register/provider/gcp/subscriber/PushSubscription.java
    + 56
    − 29
    View file @ 566aae73
    ......@@ -23,33 +23,52 @@ import com.google.pubsub.v1.ExpirationPolicy;
    import com.google.pubsub.v1.ProjectSubscriptionName;
    import com.google.pubsub.v1.ProjectTopicName;
    import com.google.pubsub.v1.PushConfig;
    import java.io.IOException;
    import java.util.Objects;
    import java.util.concurrent.TimeUnit;
    import javax.annotation.PostConstruct;
    import javax.annotation.PreDestroy;
    import lombok.extern.slf4j.Slf4j;
    import org.opengroup.osdu.core.common.model.tenant.TenantInfo;
    import org.opengroup.osdu.register.utils.AppServiceConfig;
    import org.opengroup.osdu.register.subscriber.model.Subscription;
    import org.opengroup.osdu.register.utils.AppServiceConfig;
    import org.springframework.beans.factory.annotation.Autowired;
    import org.springframework.stereotype.Component;
    import java.io.IOException;
    @Slf4j
    @Component
    public class PushSubscription {
    private static final int ACK_DEADLINE_SECONDS = 60;
    private static final int MESSAGE_RETENTION_SECONDS = 432000;
    private static final int SUBSCRIPTION_EXPIRATION_SECONDS = 31540000;
    private static final int AWAIT_TERMINATION_DURATION = 20;
    @Autowired
    private AppServiceConfig serviceConfig;
    private SubscriptionAdminClient subscriptionAdminClient;
    @PostConstruct
    public void setUpSubscriptionAdminClient() throws IOException {
    try {
    this.subscriptionAdminClient = SubscriptionAdminClient.create();
    } catch (IOException e) {
    log.error("Exception occurred while creating subscription admin client", e);
    throw e;
    }
    }
    public Subscription create(Subscription subscription, TenantInfo tenant,
    String masterGcpId, String serviceIdentityEmail)
    throws IOException {
    String masterGcpId, String serviceIdentityEmail)
    throws IOException {
    createPushSubscription(
    tenant.getProjectId(),
    masterGcpId,
    subscription.getTopic(),
    subscription.getNotificationId(),
    serviceConfig.getRecordsChangePubsubEndpoint(),
    serviceIdentityEmail);
    tenant.getProjectId(),
    masterGcpId,
    subscription.getTopic(),
    subscription.getNotificationId(),
    serviceConfig.getRecordsChangePubsubEndpoint(),
    serviceIdentityEmail);
    return subscription;
    }
    ......@@ -65,25 +84,33 @@ public class PushSubscription {
    }
    private com.google.pubsub.v1.Subscription createPushSubscription(String tenantProjectId, String servicesProjectId, String topicId,
    String subscriptionId, String url, String serviceIdentityEmail) throws IOException {
    try (SubscriptionAdminClient subscriptionAdminClient = SubscriptionAdminClient.create()) {
    ProjectTopicName topicName = ProjectTopicName.of(tenantProjectId, topicId);
    ProjectSubscriptionName subscriptionName = ProjectSubscriptionName.of(servicesProjectId, subscriptionId);
    String subscriptionId, String url, String serviceIdentityEmail) {
    ProjectTopicName topicName = ProjectTopicName.of(tenantProjectId, topicId);
    ProjectSubscriptionName subscriptionName = ProjectSubscriptionName.of(servicesProjectId, subscriptionId);
    PushConfig config = PushConfig.newBuilder()
    .setPushEndpoint(url)
    .setOidcToken(PushConfig.OidcToken.newBuilder().setServiceAccountEmail(serviceIdentityEmail).build())
    .build();
    return this.subscriptionAdminClient.createSubscription(com.google.pubsub.v1.Subscription.newBuilder()
    .setName(subscriptionName.toString())
    .setTopic(topicName.toString())
    .setPushConfig(config)
    .setExpirationPolicy(ExpirationPolicy.newBuilder().setTtl(
    Duration.newBuilder().setSeconds(SUBSCRIPTION_EXPIRATION_SECONDS).build()).build())
    .setMessageRetentionDuration(Duration.newBuilder().setSeconds(
    MESSAGE_RETENTION_SECONDS).build())
    .setAckDeadlineSeconds(ACK_DEADLINE_SECONDS)
    .build());
    }
    PushConfig config = PushConfig.newBuilder()
    .setPushEndpoint(url)
    .setOidcToken(PushConfig.OidcToken.newBuilder().setServiceAccountEmail(serviceIdentityEmail).build())
    .build();
    return subscriptionAdminClient.createSubscription(com.google.pubsub.v1.Subscription.newBuilder()
    .setName(subscriptionName.toString())
    .setTopic(topicName.toString())
    .setPushConfig(config)
    .setExpirationPolicy(ExpirationPolicy.newBuilder().setTtl(
    Duration.newBuilder().setSeconds(SUBSCRIPTION_EXPIRATION_SECONDS).build()).build())
    .setMessageRetentionDuration(Duration.newBuilder().setSeconds(
    MESSAGE_RETENTION_SECONDS).build())
    .setAckDeadlineSeconds(ACK_DEADLINE_SECONDS)
    .build());
    @PreDestroy
    public void shutDownSubscriptionAdminClient() throws InterruptedException {
    if (Objects.nonNull(this.subscriptionAdminClient)) {
    log.debug("Shutting down publisher on exit, await termination duration: {} seconds", AWAIT_TERMINATION_DURATION);
    subscriptionAdminClient.shutdown();
    subscriptionAdminClient.awaitTermination(AWAIT_TERMINATION_DURATION, TimeUnit.SECONDS);
    }
    }
    }
    provider/register-gcp/src/main/resources/application.properties
    + 2
    − 0
    View file @ 566aae73
    ......@@ -7,6 +7,8 @@ JAVA_GC_OPTS=-XX:+UseG1GC -XX:+UseStringDeduplication -XX:InitiatingHeapOccupanc
    KEY_RING=csqp
    KMS_KEY=registerService
    SERVICE_IDENTITY=de-notification-service
    CRON_JOB_EXPECTED_IP=0:0:0:0:0:0:0:1
    ACCEPT_HTTP=true
    INTEGRATION_TEST_AUDIENCES=${GOOGLE_AUDIENCES}
    SUBSCRIBER_SECRET=7a786376626e
    ......
    0% or .
    You are about to add 0 people to the discussion. Proceed with caution.
    Finish editing this message first!
    Please register or to comment