From 43660a4dcb4fe55c8acb11e0a198e21ca2a92a6a Mon Sep 17 00:00:00 2001 From: Paul-Christian Volkmer Date: Mon, 29 Apr 2024 16:45:25 +0200 Subject: [PATCH 01/12] chore: mark as snapshot version --- build.gradle.kts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle.kts b/build.gradle.kts index 93642f3..b8da6db 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -11,7 +11,7 @@ plugins { } group = "de.ukw.ccc" -version = "0.9.4" +version = "0.9-SNAPSHOT" var versions = mapOf( "bwhc-dto-java" to "0.2.0", From 01446bdece9478c71c829111f985264ebd925c2b Mon Sep 17 00:00:00 2001 From: Paul-Christian Volkmer Date: Mon, 22 Apr 2024 19:30:46 +0200 Subject: [PATCH 02/12] chore: update GitHub workflow actions --- .github/workflows/deploy.yml | 2 +- .github/workflows/test.yml | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 82e206b..b703b73 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -21,7 +21,7 @@ jobs: uses: docker/setup-buildx-action@v3 - name: Login to Docker Hub - uses: docker/login-action@v2 + uses: docker/login-action@v3 with: registry: ghcr.io username: ${{ github.actor }} diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index b1fff34..8d2ac4e 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -12,7 +12,7 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - - uses: actions/setup-java@v3 + - uses: actions/setup-java@v4 with: java-version: '21' distribution: 'temurin' @@ -27,7 +27,7 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - - uses: actions/setup-java@v3 + - uses: actions/setup-java@v4 with: java-version: '21' distribution: 'temurin' From dad3ea80ee3f532eb8a92e17765802142dda7040 Mon Sep 17 00:00:00 2001 From: Paul-Christian Volkmer Date: Wed, 24 Apr 2024 12:06:08 +0200 Subject: [PATCH 03/12] chore: update integration test dependency This mitigates CVE-204-26308 and CVE-2024-25710 --- build.gradle.kts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/build.gradle.kts b/build.gradle.kts index b8da6db..8e920b9 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -85,6 +85,8 @@ dependencies { testImplementation("org.mockito.kotlin:mockito-kotlin:${versions["mockito-kotlin"]}") integrationTestImplementation("org.testcontainers:junit-jupiter") integrationTestImplementation("org.testcontainers:postgresql") + // Override dependency version from org.testcontainers:junit-jupiter - CVE-204-26308, CVE-2024-25710 + integrationTestImplementation("org.apache.commons:commons-compress:1.26.1") } tasks.withType { From 1b4585d60123fd5102e8cf315b75208bd10a477b Mon Sep 17 00:00:00 2001 From: Paul-Christian Volkmer Date: Thu, 25 Apr 2024 09:10:33 +0200 Subject: [PATCH 04/12] docs: fix CVE number in dependency comment --- build.gradle.kts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle.kts b/build.gradle.kts index 8e920b9..0e58144 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -85,7 +85,7 @@ dependencies { testImplementation("org.mockito.kotlin:mockito-kotlin:${versions["mockito-kotlin"]}") integrationTestImplementation("org.testcontainers:junit-jupiter") integrationTestImplementation("org.testcontainers:postgresql") - // Override dependency version from org.testcontainers:junit-jupiter - CVE-204-26308, CVE-2024-25710 + // Override dependency version from org.testcontainers:junit-jupiter - CVE-2024-26308, CVE-2024-25710 integrationTestImplementation("org.apache.commons:commons-compress:1.26.1") } From bb0bbf5a280c167bb187a921978eda40ca7ecd49 Mon Sep 17 00:00:00 2001 From: Paul-Christian Volkmer Date: Thu, 25 Apr 2024 09:33:35 +0200 Subject: [PATCH 05/12] chore: update webjars-locator dependency --- build.gradle.kts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle.kts b/build.gradle.kts index 0e58144..862cd47 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -70,7 +70,7 @@ dependencies { implementation("ca.uhn.hapi.fhir:hapi-fhir-structures-r4:${versions["hapi-fhir"]}") implementation("org.apache.httpcomponents.client5:httpclient5:${versions["httpclient5"]}") implementation("com.jayway.jsonpath:json-path") - implementation("org.webjars:webjars-locator:0.50") + implementation("org.webjars:webjars-locator:0.52") implementation("org.webjars.npm:echarts:${versions["echarts"]}") implementation("org.webjars.npm:htmx.org:${versions["htmx.org"]}") runtimeOnly("org.mariadb.jdbc:mariadb-java-client") From 4d5c0ce1fbdc3057ad39e9e0e82689a7f442e38a Mon Sep 17 00:00:00 2001 From: Paul-Christian Volkmer Date: Mon, 6 May 2024 10:03:36 +0200 Subject: [PATCH 06/12] chore: remove println --- .../dev/dnpm/etl/processor/web/StatisticsRestController.kt | 1 - 1 file changed, 1 deletion(-) diff --git a/src/main/kotlin/dev/dnpm/etl/processor/web/StatisticsRestController.kt b/src/main/kotlin/dev/dnpm/etl/processor/web/StatisticsRestController.kt index 74ae238..daa6af3 100644 --- a/src/main/kotlin/dev/dnpm/etl/processor/web/StatisticsRestController.kt +++ b/src/main/kotlin/dev/dnpm/etl/processor/web/StatisticsRestController.kt @@ -134,7 +134,6 @@ class StatisticsRestController( @GetMapping(path = ["events"], produces = [MediaType.TEXT_EVENT_STREAM_VALUE]) fun updater(): Flux> { return statisticsUpdateProducer.asFlux().flatMap { - println(it) Flux.fromIterable( listOf( ServerSentEvent.builder() From c922e27758186d410a3c5e2148d3a1b6018cfa29 Mon Sep 17 00:00:00 2001 From: Paul-Christian Volkmer Date: Mon, 6 May 2024 10:43:03 +0200 Subject: [PATCH 07/12] fix: handle null values in MtbFile This should not occur but if, it should not result in NPE except for * Patient * Consent * Episode --- .../etl/processor/pseudonym/extensions.kt | 46 +++++++------- .../etl/processor/pseudonym/ExtensionsTest.kt | 61 +++++++++++++++++++ 2 files changed, 84 insertions(+), 23 deletions(-) diff --git a/src/main/kotlin/dev/dnpm/etl/processor/pseudonym/extensions.kt b/src/main/kotlin/dev/dnpm/etl/processor/pseudonym/extensions.kt index b0221bc..ef25787 100644 --- a/src/main/kotlin/dev/dnpm/etl/processor/pseudonym/extensions.kt +++ b/src/main/kotlin/dev/dnpm/etl/processor/pseudonym/extensions.kt @@ -1,7 +1,7 @@ /* * This file is part of ETL-Processor * - * Copyright (c) 2023 Comprehensive Cancer Center Mainfranken, Datenintegrationszentrum Philipps-Universität Marburg and Contributors + * Copyright (c) 2024 Comprehensive Cancer Center Mainfranken, Datenintegrationszentrum Philipps-Universität Marburg and Contributors * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU Affero General Public License as published @@ -31,31 +31,31 @@ import org.apache.commons.codec.digest.DigestUtils infix fun MtbFile.pseudonymizeWith(pseudonymizeService: PseudonymizeService) { val patientPseudonym = pseudonymizeService.patientPseudonym(this.patient.id) - this.episode.patient = patientPseudonym - this.carePlans.forEach { it.patient = patientPseudonym } + this.episode?.patient = patientPseudonym + this.carePlans?.forEach { it.patient = patientPseudonym } this.patient.id = patientPseudonym - this.claims.forEach { it.patient = patientPseudonym } - this.consent.patient = patientPseudonym - this.claimResponses.forEach { it.patient = patientPseudonym } - this.diagnoses.forEach { it.patient = patientPseudonym } - this.ecogStatus.forEach { it.patient = patientPseudonym } - this.familyMemberDiagnoses.forEach { it.patient = patientPseudonym } - this.geneticCounsellingRequests.forEach { it.patient = patientPseudonym } - this.histologyReevaluationRequests.forEach { it.patient = patientPseudonym } - this.histologyReports.forEach { + this.claims?.forEach { it.patient = patientPseudonym } + this.consent?.patient = patientPseudonym + this.claimResponses?.forEach { it.patient = patientPseudonym } + this.diagnoses?.forEach { it.patient = patientPseudonym } + this.ecogStatus?.forEach { it.patient = patientPseudonym } + this.familyMemberDiagnoses?.forEach { it.patient = patientPseudonym } + this.geneticCounsellingRequests?.forEach { it.patient = patientPseudonym } + this.histologyReevaluationRequests?.forEach { it.patient = patientPseudonym } + this.histologyReports?.forEach { it.patient = patientPseudonym - it.tumorMorphology.patient = patientPseudonym + it.tumorMorphology?.patient = patientPseudonym } - this.lastGuidelineTherapies.forEach { it.patient = patientPseudonym } - this.molecularPathologyFindings.forEach { it.patient = patientPseudonym } - this.molecularTherapies.forEach { molecularTherapy -> molecularTherapy.history.forEach { it.patient = patientPseudonym } } - this.ngsReports.forEach { it.patient = patientPseudonym } - this.previousGuidelineTherapies.forEach { it.patient = patientPseudonym } - this.rebiopsyRequests.forEach { it.patient = patientPseudonym } - this.recommendations.forEach { it.patient = patientPseudonym } - this.responses.forEach { it.patient = patientPseudonym } - this.studyInclusionRequests.forEach { it.patient = patientPseudonym } - this.specimens.forEach { it.patient = patientPseudonym } + this.lastGuidelineTherapies?.forEach { it.patient = patientPseudonym } + this.molecularPathologyFindings?.forEach { it.patient = patientPseudonym } + this.molecularTherapies?.forEach { molecularTherapy -> molecularTherapy.history.forEach { it.patient = patientPseudonym } } + this.ngsReports?.forEach { it.patient = patientPseudonym } + this.previousGuidelineTherapies?.forEach { it.patient = patientPseudonym } + this.rebiopsyRequests?.forEach { it.patient = patientPseudonym } + this.recommendations?.forEach { it.patient = patientPseudonym } + this.responses?.forEach { it.patient = patientPseudonym } + this.studyInclusionRequests?.forEach { it.patient = patientPseudonym } + this.specimens?.forEach { it.patient = patientPseudonym } } /** diff --git a/src/test/kotlin/dev/dnpm/etl/processor/pseudonym/ExtensionsTest.kt b/src/test/kotlin/dev/dnpm/etl/processor/pseudonym/ExtensionsTest.kt index c980635..d8c7813 100644 --- a/src/test/kotlin/dev/dnpm/etl/processor/pseudonym/ExtensionsTest.kt +++ b/src/test/kotlin/dev/dnpm/etl/processor/pseudonym/ExtensionsTest.kt @@ -134,4 +134,65 @@ class ExtensionsTest { .isEqualTo("TESTDOMAIN44e20a53bbbf9f3ae39626d05df7014dcd77d6098") } + @Test + fun shouldNotThrowExceptionOnNullValues(@Mock pseudonymizeService: PseudonymizeService) { + doAnswer { + it.arguments[0] + "PSEUDO-ID" + }.whenever(pseudonymizeService).patientPseudonym(ArgumentMatchers.anyString()) + + doAnswer { + "TESTDOMAIN" + }.whenever(pseudonymizeService).prefix() + + val mtbFile = MtbFile.builder() + .withPatient( + Patient.builder() + .withId("1") + .withBirthDate("2000-08-08") + .withGender(Patient.Gender.MALE) + .build() + ) + .withConsent( + Consent.builder() + .withId("1") + .withStatus(Consent.Status.ACTIVE) + .withPatient("123") + .build() + ) + .withEpisode( + Episode.builder() + .withId("1") + .withPatient("1") + .withPeriod(PeriodStart("2023-08-08")) + .build() + ) + .withClaims(null) + .withDiagnoses(null) + .withCarePlans(null) + .withClaimResponses(null) + .withEcogStatus(null) + .withFamilyMemberDiagnoses(null) + .withGeneticCounsellingRequests(null) + .withHistologyReevaluationRequests(null) + .withHistologyReports(null) + .withLastGuidelineTherapies(null) + .withMolecularPathologyFindings(null) + .withMolecularTherapies(null) + .withNgsReports(null) + .withPreviousGuidelineTherapies(null) + .withRebiopsyRequests(null) + .withRecommendations(null) + .withResponses(null) + .withStudyInclusionRequests(null) + .withSpecimens(null) + .build() + + mtbFile.pseudonymizeWith(pseudonymizeService) + mtbFile.anonymizeContentWith(pseudonymizeService) + + + assertThat(mtbFile.episode.id).isNotNull() + } + } \ No newline at end of file From 041bf459efce5663bd769ce460dcb329ea85d994 Mon Sep 17 00:00:00 2001 From: Paul-Christian Volkmer Date: Wed, 15 May 2024 17:27:04 +0200 Subject: [PATCH 08/12] fix: add missing 'fatal' severity --- .../etl/processor/monitoring/ReportService.kt | 1 + src/main/resources/templates/report.html | 1 + .../processor/services/ReportServiceTest.kt | 19 +++++++++++-------- 3 files changed, 13 insertions(+), 8 deletions(-) diff --git a/src/main/kotlin/dev/dnpm/etl/processor/monitoring/ReportService.kt b/src/main/kotlin/dev/dnpm/etl/processor/monitoring/ReportService.kt index ccbbe1c..97ecd05 100644 --- a/src/main/kotlin/dev/dnpm/etl/processor/monitoring/ReportService.kt +++ b/src/main/kotlin/dev/dnpm/etl/processor/monitoring/ReportService.kt @@ -57,6 +57,7 @@ class ReportService( data class Issue(val severity: Severity, val message: String) enum class Severity(@JsonValue val value: String) { + FATAL("fatal"), ERROR("error"), WARNING("warning"), INFO("info") diff --git a/src/main/resources/templates/report.html b/src/main/resources/templates/report.html index 6f89345..07f987c 100644 --- a/src/main/resources/templates/report.html +++ b/src/main/resources/templates/report.html @@ -55,6 +55,7 @@ [[ ${issue.severity} ]] [[ ${issue.severity} ]] [[ ${issue.severity} ]] + [[ ${issue.severity} ]] [[ ${issue.message} ]] diff --git a/src/test/kotlin/dev/dnpm/etl/processor/services/ReportServiceTest.kt b/src/test/kotlin/dev/dnpm/etl/processor/services/ReportServiceTest.kt index f0f6230..349202a 100644 --- a/src/test/kotlin/dev/dnpm/etl/processor/services/ReportServiceTest.kt +++ b/src/test/kotlin/dev/dnpm/etl/processor/services/ReportServiceTest.kt @@ -43,20 +43,23 @@ class ReportServiceTest { "issues": [ { "severity": "info", "message": "Info Message" }, { "severity": "warning", "message": "Warning Message" }, - { "severity": "error", "message": "Error Message" } + { "severity": "error", "message": "Error Message" }, + { "severity": "fatal", "message": "Fatal Message" } ] } """.trimIndent() val actual = this.reportService.deserialize(json) - assertThat(actual).hasSize(3) - assertThat(actual[0].severity).isEqualTo(ReportService.Severity.ERROR) - assertThat(actual[0].message).isEqualTo("Error Message") - assertThat(actual[1].severity).isEqualTo(ReportService.Severity.WARNING) - assertThat(actual[1].message).isEqualTo("Warning Message") - assertThat(actual[2].severity).isEqualTo(ReportService.Severity.INFO) - assertThat(actual[2].message).isEqualTo("Info Message") + assertThat(actual).hasSize(4) + assertThat(actual[0].severity).isEqualTo(ReportService.Severity.FATAL) + assertThat(actual[0].message).isEqualTo("Fatal Message") + assertThat(actual[1].severity).isEqualTo(ReportService.Severity.ERROR) + assertThat(actual[1].message).isEqualTo("Error Message") + assertThat(actual[2].severity).isEqualTo(ReportService.Severity.WARNING) + assertThat(actual[2].message).isEqualTo("Warning Message") + assertThat(actual[3].severity).isEqualTo(ReportService.Severity.INFO) + assertThat(actual[3].message).isEqualTo("Info Message") } @Test From ca1e73a0b52462eedff946dc31c07713d805d340 Mon Sep 17 00:00:00 2001 From: Paul-Christian Volkmer Date: Thu, 16 May 2024 17:59:22 +0200 Subject: [PATCH 09/12] chore: update bwhc-dto-java dependency --- build.gradle.kts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle.kts b/build.gradle.kts index 862cd47..5df6a6e 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -14,7 +14,7 @@ group = "de.ukw.ccc" version = "0.9-SNAPSHOT" var versions = mapOf( - "bwhc-dto-java" to "0.2.0", + "bwhc-dto-java" to "0.3.0", "hapi-fhir" to "6.10.5", "httpclient5" to "5.2.3", "mockito-kotlin" to "5.3.1", From ce1489d9a1bb96ff1b350b4092f12ba963a18e39 Mon Sep 17 00:00:00 2001 From: Paul-Christian Volkmer Date: Mon, 27 May 2024 09:08:08 +0200 Subject: [PATCH 10/12] chore: update spring boot dependencies --- build.gradle.kts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle.kts b/build.gradle.kts index 5df6a6e..8ac7349 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -4,7 +4,7 @@ import org.springframework.boot.gradle.tasks.bundling.BootBuildImage plugins { war - id("org.springframework.boot") version "3.2.5" + id("org.springframework.boot") version "3.2.6" id("io.spring.dependency-management") version "1.1.4" kotlin("jvm") version "1.9.23" kotlin("plugin.spring") version "1.9.23" From 070100eba05fd6a0a2633dd9e4c5cd3f9c0ef7a2 Mon Sep 17 00:00:00 2001 From: Paul-Christian Volkmer Date: Mon, 27 May 2024 12:33:15 +0200 Subject: [PATCH 11/12] chore: update spring dependency-management plugin --- build.gradle.kts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle.kts b/build.gradle.kts index 8ac7349..4c26129 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -5,7 +5,7 @@ import org.springframework.boot.gradle.tasks.bundling.BootBuildImage plugins { war id("org.springframework.boot") version "3.2.6" - id("io.spring.dependency-management") version "1.1.4" + id("io.spring.dependency-management") version "1.1.5" kotlin("jvm") version "1.9.23" kotlin("plugin.spring") version "1.9.23" } From 8b194e7212e58d72788a1ac48203c478c3808a6a Mon Sep 17 00:00:00 2001 From: Paul-Christian Volkmer Date: Mon, 27 May 2024 12:33:40 +0200 Subject: [PATCH 12/12] chore: update kotlin version --- build.gradle.kts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/build.gradle.kts b/build.gradle.kts index 4c26129..dcbeed4 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -6,8 +6,8 @@ plugins { war id("org.springframework.boot") version "3.2.6" id("io.spring.dependency-management") version "1.1.5" - kotlin("jvm") version "1.9.23" - kotlin("plugin.spring") version "1.9.23" + kotlin("jvm") version "1.9.24" + kotlin("plugin.spring") version "1.9.24" } group = "de.ukw.ccc"