pcvolkmer/etl-processor
1
0
Fork 0
mirror of https://github.com/pcvolkmer/etl-processor.git synced 2025-07-01 14:12:55 +00:00
Code Issues Releases Activity

feat: check consent via gICS before process mtb file

Browse Source
This commit is contained in:
Jakub Lidke
2025-04-28 11:44:26 +02:00
parent 464c8b8c1d
commit b27670535f
21 changed files with 562 additions and 103 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
build.gradle.kts
View File

@ -2,6 +2,8 @@ import org.gradle.api.tasks.testing.logging.TestLogEvent
import org.jetbrains.kotlin.gradle.dsl.JvmTarget
import org.jetbrains.kotlin.gradle.tasks.KotlinCompile
import org.springframework.boot.gradle.tasks.bundling.BootBuildImage
import java.text.SimpleDateFormat
import java.util.*
plugins {
war
@ -133,6 +135,7 @@ tasks.jacocoTestReport {
}
}
tasks.named<BootBuildImage>("bootBuildImage") {
imageName.set("ghcr.io/ccc-mf/etl-processor")
@ -143,7 +146,8 @@ tasks.named<BootBuildImage>("bootBuildImage") {
environment.set(environment.get() + mapOf(
// Enable this line to embed CA Certs into image on build time
//"BP_EMBED_CERTS" to "true",
"BP_EMBED_CERTS" to "true",
"BP_OCI_CREATED" to SimpleDateFormat("MM-dd-yyyy_hh-mm").format(Date()),
"BP_OCI_SOURCE" to "https://github.com/CCC-MF/etl-processor",
"BP_OCI_LICENSES" to "AGPLv3",
"BP_OCI_DESCRIPTION" to "ETL Processor for bwHC MTB files"

44
dev-compose.yml
View File

@ -16,6 +16,11 @@ services:
KAFKA_CFG_AUTO_CREATE_TOPICS_ENABLE: true
KAFKA_CFG_CONTROLLER_QUORUM_VOTERS: 0@kafka:9093
KAFKA_CFG_CONTROLLER_LISTENER_NAMES: CONTROLLER
healthcheck:
test: kafka-topics --bootstrap-server kafka:9092 --list
interval: 30s
timeout: 10s
retries: 3
## Use AKHQ as Kafka web frontend
akhq:
@ -53,4 +58,41 @@ services:
# environment:
# POSTGRES_DB: dev
# POSTGRES_USER: dev
# POSTGRES_PASSWORD: dev
# POSTGRES_PASSWORD: dev
mysql:
image: mysql:8
container_name: gics-mysql
restart: unless-stopped
environment:
MYSQL_ROOT_PASSWORD: root
TZ: Europe/Berlin
ports:
- "3306:3306"
volumes:
- ./sqls:/docker-entrypoint-initdb.d
command: --max_allowed_packet=20M --default-time-zone=Europe/Berlin
gics:
image: registry.diz.uni-marburg.de/ths/gics:2023.1.3
container_name: gics-wildfly
restart: unless-stopped
ports:
- "8090:8080"
- "127.0.0.1:9992:9990"
extra_hosts:
- "host.docker.internal:host-gateway"
depends_on:
- mysql
consent-data-loader:
image: confluentinc/cp-kafkacat:7.1.12
entrypoint: ["/bin/bash", "-c"]
command: >
"kafkacat -b kafka:19092 -K: -t consent-json -P -l /data/consent-data.ndjson"
volumes:
- ./consent-data.ndjson:/data/consent-data.ndjson:ro
depends_on:
kafka:
condition: service_healthy

74
dev/docker-compose.dev.yml
View File

@ -2,31 +2,55 @@ version: '3.7'
services:
zoo1:
image: zookeeper:3.8.0
hostname: zoo1
zoo:
image: zookeeper:3.9.2
restart: unless-stopped
ports:
- "2181:2181"
environment:
ZOO_MY_ID: 1
ZOO_PORT: 2181
ZOO_SERVERS: server.1=zoo1:2888:3888;2181
ZOO_SERVERS: server.1=zoo:2888:3888;2181
kafka1:
image: confluentinc/cp-kafka:7.2.1
hostname: kafka1
kafka:
image: confluentinc/cp-kafka:7.6.1
ports:
- "9092:9092"
environment:
KAFKA_ADVERTISED_LISTENERS: LISTENER_DOCKER_INTERNAL://kafka1:19092,LISTENER_DOCKER_EXTERNAL://${DOCKER_HOST_IP:-127.0.0.1}:9092
KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: LISTENER_DOCKER_INTERNAL:PLAINTEXT,LISTENER_DOCKER_EXTERNAL:PLAINTEXT
KAFKA_ADVERTISED_LISTENERS: LISTENER_DOCKER_INTERNAL://kafka:19092,LISTENER_DOCKER_EXTERNAL://172.17.0.1:9093,LISTENER_EXTERNAL://127.0.0.1:9092
KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: LISTENER_DOCKER_INTERNAL:PLAINTEXT,LISTENER_DOCKER_EXTERNAL:PLAINTEXT,LISTENER_EXTERNAL:PLAINTEXT
KAFKA_INTER_BROKER_LISTENER_NAME: LISTENER_DOCKER_INTERNAL
KAFKA_ZOOKEEPER_CONNECT: "zoo1:2181"
KAFKA_ZOOKEEPER_CONNECT: zoo:2181
KAFKA_BROKER_ID: 1
KAFKA_LOG4J_LOGGERS: "kafka.controller=INFO,kafka.producer.async.DefaultEventHandler=INFO,state.change.logger=INFO"
KAFKA_LOG4J_LOGGERS: kafka.controller=INFO,kafka.producer.async.DefaultEventHandler=INFO,state.change.logger=INFO
KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
KAFKA_MESSAGE_MAX_BYTES: 5242880
KAFKA_REPLICA_FETCH_MAX_BYTES: 5242880
KAFKA_COMPRESSION_TYPE: gzip
depends_on:
- zoo1
- zoo
healthcheck:
test: kafka-topics --bootstrap-server kafka:9092 --list
interval: 30s
timeout: 10s
retries: 3
akhq:
image: tchiotludo/akhq:0.25.0
environment:
AKHQ_CONFIGURATION: |
akhq:
ui-options:
topic.show-all-consumer-groups: true
topic-data.sort: NEWEST
connections:
docker-kafka-server:
properties:
bootstrap.servers: "kafka:19092"
ports:
- "9000:8080"
depends_on:
- kafka
kafka-rest-proxy:
image: confluentinc/cp-kafka-rest:7.2.1
@ -40,8 +64,8 @@ services:
KAFKA_REST_HOST_NAME: kafka-rest-proxy
KAFKA_REST_BOOTSTRAP_SERVERS: PLAINTEXT://kafka1:19092
depends_on:
- zoo1
- kafka1
- zoo
- kafka
kafka-connect:
image: confluentinc/cp-kafka-connect:7.2.1
@ -67,24 +91,6 @@ services:
#volumes:
# - ./connectors:/etc/kafka-connect/jars/
depends_on:
- zoo1
- kafka1
- zoo
- kafka
- kafka-rest-proxy
akhq:
image: tchiotludo/akhq:0.21.0
environment:
AKHQ_CONFIGURATION: |
akhq:
connections:
docker-kafka-server:
properties:
bootstrap.servers: "kafka1:19092"
connect:
- name: "kafka-connect"
url: "http://kafka-connect:8083"
ports:
- "8084:8080"
depends_on:
- kafka1
- kafka-connect

23
src/integrationTest/kotlin/dev/dnpm/etl/processor/input/MtbFileRestControllerTest.kt
View File

@ -22,7 +22,11 @@ package dev.dnpm.etl.processor.input
import com.fasterxml.jackson.databind.ObjectMapper
import de.ukw.ccc.bwhc.dto.*
import dev.dnpm.etl.processor.anyValueClass
import dev.dnpm.etl.processor.config.AppFhirConfig
import dev.dnpm.etl.processor.config.AppSecurityConfiguration
import dev.dnpm.etl.processor.consent.ConsentCheckedIgnored
import dev.dnpm.etl.processor.consent.ConsentStatus
import dev.dnpm.etl.processor.consent.ICheckConsent
import dev.dnpm.etl.processor.security.TokenRepository
import dev.dnpm.etl.processor.security.UserRoleRepository
import dev.dnpm.etl.processor.services.RequestProcessor
@ -30,11 +34,9 @@ import org.junit.jupiter.api.BeforeEach
import org.junit.jupiter.api.Nested
import org.junit.jupiter.api.Test
import org.junit.jupiter.api.extension.ExtendWith
import org.mockito.Mockito
import org.mockito.junit.jupiter.MockitoExtension
import org.mockito.kotlin.any
import org.mockito.kotlin.never
import org.mockito.kotlin.times
import org.mockito.kotlin.verify
import org.mockito.kotlin.*
import org.springframework.beans.factory.annotation.Autowired
import org.springframework.boot.test.autoconfigure.web.servlet.WebMvcTest
import org.springframework.boot.test.mock.mockito.MockBean
@ -54,7 +56,8 @@ import org.springframework.test.web.servlet.post
@ContextConfiguration(
classes = [
MtbFileRestController::class,
AppSecurityConfiguration::class
AppSecurityConfiguration::class,
ConsentCheckedIgnored::class, ICheckConsent::class
]
)
@MockBean(TokenRepository::class, RequestProcessor::class)
@ -63,7 +66,8 @@ import org.springframework.test.web.servlet.post
"app.pseudonymize.generator=BUILDIN",
"app.security.admin-user=admin",
"app.security.admin-password={noop}very-secret",
"app.security.enable-tokens=true"
"app.security.enable-tokens=true",
"app.consent.gics.enabled=false"
]
)
class MtbFileRestControllerTest {
@ -141,7 +145,7 @@ class MtbFileRestControllerTest {
status { isAccepted() }
}
verify(requestProcessor, times(1)).processDeletion(anyValueClass())
verify(requestProcessor, times(1)).processDeletion(anyValueClass(), eq(ConsentStatus.IGNORED))
}
@Test
@ -152,7 +156,7 @@ class MtbFileRestControllerTest {
status { isUnauthorized() }
}
verify(requestProcessor, never()).processDeletion(anyValueClass())
verify(requestProcessor, never()).processDeletion(anyValueClass(), any())
}
@Nested
@ -163,7 +167,8 @@ class MtbFileRestControllerTest {
"app.security.admin-user=admin",
"app.security.admin-password={noop}very-secret",
"app.security.enable-tokens=true",
"app.security.enable-oidc=true"
"app.security.enable-oidc=true",
"app.consent.gics.enabled=false"
]
)
inner class WithOidcEnabled {

18
src/integrationTest/kotlin/dev/dnpm/etl/processor/pseudonym/GpasPseudonymGeneratorTest.kt
View File

@ -19,6 +19,7 @@
package dev.dnpm.etl.processor.pseudonym
import dev.dnpm.etl.processor.config.AppFhirConfig
import dev.dnpm.etl.processor.config.GPasConfigProperties
import org.assertj.core.api.Assertions.assertThat
import org.junit.jupiter.api.BeforeEach
@ -42,6 +43,7 @@ class GpasPseudonymGeneratorTest {
private lateinit var mockRestServiceServer: MockRestServiceServer
private lateinit var generator: GpasPseudonymGenerator
private lateinit var restTemplate: RestTemplate
private var appFhirConfig: AppFhirConfig = AppFhirConfig()
@BeforeEach
fun setup() {
@ -56,7 +58,8 @@ class GpasPseudonymGeneratorTest {
this.restTemplate = RestTemplate()
this.mockRestServiceServer = MockRestServiceServer.createServer(restTemplate)
this.generator = GpasPseudonymGenerator(gPasConfigProperties, retryTemplate, restTemplate)
this.generator =
GpasPseudonymGenerator(gPasConfigProperties, retryTemplate, restTemplate, appFhirConfig)
}
@Test
@ -65,7 +68,13 @@ class GpasPseudonymGeneratorTest {
method(HttpMethod.POST)
requestTo("http://localhost/ttp-fhir/fhir/gpas/\$pseudonymizeAllowCreate")
}.andRespond {
withStatus(HttpStatus.OK).body(getDummyResponseBody("1234", "test", "test1234ABCDEF567890"))
withStatus(HttpStatus.OK).body(
getDummyResponseBody(
"1234",
"test",
"test1234ABCDEF567890"
)
)
.createResponse(it)
}
@ -91,7 +100,10 @@ class GpasPseudonymGeneratorTest {
requestTo("http://localhost/ttp-fhir/fhir/gpas/\$pseudonymizeAllowCreate")
}.andRespond {
withStatus(HttpStatus.FOUND)
.header(HttpHeaders.LOCATION, "https://localhost/ttp-fhir/fhir/gpas/\$pseudonymizeAllowCreate")
.header(
HttpHeaders.LOCATION,
"https://localhost/ttp-fhir/fhir/gpas/\$pseudonymizeAllowCreate"
)
.createResponse(it)
}

9
src/main/java/dev/dnpm/etl/processor/consent/ConsentCheckedIgnored.java Normal file
View File

@ -0,0 +1,9 @@
package dev.dnpm.etl.processor.consent;
public class ConsentCheckedIgnored implements ICheckConsent{
@Override
public ConsentStatus isConsented(String personIdentifierValue) {
return ConsentStatus.IGNORED;
}
}

8
src/main/java/dev/dnpm/etl/processor/consent/ConsentStatus.java Normal file
View File

@ -0,0 +1,8 @@
package dev.dnpm.etl.processor.consent;
public enum ConsentStatus {
CONSENTED,
CONSENT_MISSING,
FAILED_TO_ASK,
IGNORED, CONSENT_REJECTED
}

157
src/main/java/dev/dnpm/etl/processor/consent/GicsConsentService.java Normal file
View File

@ -0,0 +1,157 @@
package dev.dnpm.etl.processor.consent;
import ca.uhn.fhir.context.FhirContext;
import dev.dnpm.etl.processor.config.AppFhirConfig;
import dev.dnpm.etl.processor.config.GIcsConfigProperties;
import org.apache.commons.lang3.StringUtils;
import org.hl7.fhir.r4.model.Coding;
import org.hl7.fhir.r4.model.Identifier;
import org.hl7.fhir.r4.model.Parameters;
import org.hl7.fhir.r4.model.Parameters.ParametersParameterComponent;
import org.hl7.fhir.r4.model.StringType;
import org.jetbrains.annotations.NotNull;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.http.HttpEntity;
import org.springframework.http.HttpHeaders;
import org.springframework.http.HttpMethod;
import org.springframework.http.MediaType;
import org.springframework.http.ResponseEntity;
import org.springframework.retry.TerminatedRetryException;
import org.springframework.retry.support.RetryTemplate;
import org.springframework.web.client.RestClientException;
import org.springframework.web.client.RestTemplate;
import org.springframework.web.util.UriComponentsBuilder;
public class GicsConsentService implements ICheckConsent {
private final Logger log = LoggerFactory.getLogger(GicsConsentService.class);
private final GIcsConfigProperties gIcsConfigProperties;
public final String IS_CONSENTED_PATH = "/ttp-fhir/fhir/gics/$isConsented";
private final RetryTemplate retryTemplate;
private final RestTemplate restTemplate;
private final FhirContext fhirContext;
private final HttpHeaders httpHeader;
private String url;
public GicsConsentService(GIcsConfigProperties gIcsConfigProperties,
RetryTemplate retryTemplate, RestTemplate restTemplate, AppFhirConfig appFhirConfig) {
this.gIcsConfigProperties = gIcsConfigProperties;
this.retryTemplate = retryTemplate;
this.restTemplate = restTemplate;
this.fhirContext = appFhirConfig.fhirContext();
httpHeader = buildHeader(gIcsConfigProperties.getUsername(),
gIcsConfigProperties.getPassword());
}
public String getGicsUri() {
if (url == null) {
final String gIcsBaseUri = gIcsConfigProperties.getGIcsBaseUri();
if (StringUtils.isBlank(gIcsBaseUri)) {
throw new IllegalArgumentException(
"gICS base URL is empty - should call gICS with false configuration.");
}
url = UriComponentsBuilder.fromHttpUrl(gIcsBaseUri)
.path(IS_CONSENTED_PATH)
.toUriString();
}
return url;
}
@NotNull
private static HttpHeaders buildHeader(String gPasUserName, String gPasPassword) {
var headers = new HttpHeaders();
headers.setContentType(MediaType.APPLICATION_XML);
if (StringUtils.isBlank(gPasUserName) || StringUtils.isBlank(gPasPassword)) {
return headers;
}
headers.setBasicAuth(gPasUserName, gPasPassword);
return headers;
}
public static Parameters getIsConsentedParam(GIcsConfigProperties configProperties,
String personIdentifierValue) {
var result = new Parameters();
result.addParameter(new ParametersParameterComponent().setName("personIdentifier").setValue(
new Identifier().setValue(personIdentifierValue)
.setSystem(configProperties.getPersonIdentifierSystem())));
result.addParameter(new ParametersParameterComponent().setName("domain")
.setValue(new StringType().setValue(configProperties.getConsentDomainName())));
result.addParameter(new ParametersParameterComponent().setName("policy")
.setValue(new Coding().setCode(configProperties.getPolicyCode())
.setSystem(configProperties.getPolicySystem())));
result.addParameter(new ParametersParameterComponent().setName("version")
.setValue(new StringType().setValue(configProperties.getParameterVersion())));
return result;
}
protected String getConsentStatusResponse(Parameters parameter) {
var parameterAsXml = fhirContext.newXmlParser().encodeResourceToString(parameter);
HttpEntity<String> requestEntity = new HttpEntity<>(parameterAsXml, this.httpHeader);
ResponseEntity<String> responseEntity;
try {
responseEntity = retryTemplate.execute(
ctx -> restTemplate.exchange(getGicsUri(), HttpMethod.POST, requestEntity,
String.class));
} catch (RestClientException e) {
var msg = String.format("Get consents status request failed reason: '%s",
e.getMessage());
log.error(msg);
return null;
} catch (TerminatedRetryException terminatedRetryException) {
var msg = String.format(
"Get consents status process has been terminated. termination reason: '%s",
terminatedRetryException.getMessage());
log.error(msg
);
return null;
}
if (responseEntity.getStatusCode().is2xxSuccessful()) {
return responseEntity.getBody();
} else {
var msg = String.format(
"Trusted party system reached but request failed! code: '%s' response: '%s'",
responseEntity.getStatusCode(), responseEntity.getBody());
log.error(msg);
return null;
}
}
@Override
public ConsentStatus isConsented(String personIdentifierValue) {
var parameter = GicsConsentService.getIsConsentedParam(gIcsConfigProperties,
personIdentifierValue);
var consentStatusResponse = getConsentStatusResponse(parameter);
return evaluateConsentResponse(consentStatusResponse);
}
private ConsentStatus evaluateConsentResponse(String consentStatusResponse) {
if (consentStatusResponse == null) {
return ConsentStatus.FAILED_TO_ASK;
}
var responseParameters = fhirContext.newJsonParser()
.parseResource(Parameters.class, consentStatusResponse);
var responseValue = responseParameters.getParameter("consented").getValue();
var isConsented = responseValue.castToBoolean(responseValue);
if (!isConsented.hasValue()) {
return ConsentStatus.FAILED_TO_ASK;
}
if (isConsented.booleanValue()) {
return ConsentStatus.CONSENTED;
} else {
return ConsentStatus.CONSENT_MISSING;
}
}
}

8
src/main/java/dev/dnpm/etl/processor/consent/ICheckConsent.java Normal file
View File

@ -0,0 +1,8 @@
package dev.dnpm.etl.processor.consent;
public interface ICheckConsent {
ConsentStatus isConsented(String personIdentifierValue);
}

26
src/main/java/dev/dnpm/etl/processor/pseudonym/GpasPseudonymGenerator.java
View File

@ -21,6 +21,7 @@ package dev.dnpm.etl.processor.pseudonym;
import ca.uhn.fhir.context.FhirContext;
import ca.uhn.fhir.parser.IParser;
import dev.dnpm.etl.processor.config.AppFhirConfig;
import dev.dnpm.etl.processor.config.GPasConfigProperties;
import org.apache.commons.lang3.StringUtils;
import org.hl7.fhir.r4.model.Identifier;
@ -36,7 +37,7 @@ import org.springframework.web.client.RestTemplate;
public class GpasPseudonymGenerator implements Generator {
private final static FhirContext r4Context = FhirContext.forR4();
private final FhirContext r4Context;
private final String gPasUrl;
private final String psnTargetDomain;
private final HttpHeaders httpHeader;
@ -45,11 +46,13 @@ public class GpasPseudonymGenerator implements Generator {
private final RestTemplate restTemplate;
public GpasPseudonymGenerator(GPasConfigProperties gpasCfg, RetryTemplate retryTemplate, RestTemplate restTemplate) {
public GpasPseudonymGenerator(GPasConfigProperties gpasCfg, RetryTemplate retryTemplate,
RestTemplate restTemplate, AppFhirConfig appFhirConfig) {
this.retryTemplate = retryTemplate;
this.restTemplate = restTemplate;
this.gPasUrl = gpasCfg.getUri();
this.psnTargetDomain = gpasCfg.getTarget();
this.r4Context = appFhirConfig.fhirContext();
httpHeader = getHttpHeaders(gpasCfg.getUsername(), gpasCfg.getPassword());
log.debug(String.format("%s has been initialized", this.getClass().getName()));
@ -61,7 +64,7 @@ public class GpasPseudonymGenerator implements Generator {
var gPasRequestBody = getGpasRequestBody(id);
var responseEntity = getGpasPseudonym(gPasRequestBody);
var gPasPseudonymResult = (Parameters) r4Context.newJsonParser()
.parseResource(responseEntity.getBody());
.parseResource(responseEntity.getBody());
return unwrapPseudonym(gPasPseudonymResult);
}
@ -75,9 +78,9 @@ public class GpasPseudonymGenerator implements Generator {
}
final var identifier = (Identifier) parameters.get().getPart().stream()
.filter(a -> a.getName().equals("pseudonym"))
.findFirst()
.orElseGet(ParametersParameterComponent::new).getValue();
.filter(a -> a.getName().equals("pseudonym"))
.findFirst()
.orElseGet(ParametersParameterComponent::new).getValue();
// pseudonym
return sanitizeValue(identifier.getValue());
@ -97,7 +100,6 @@ public class GpasPseudonymGenerator implements Generator {
return psnValue.replaceAll(forbiddenCharsRegex, "_");
}
@NotNull
protected ResponseEntity<String> getGpasPseudonym(String gPasRequestBody) {
@ -106,8 +108,8 @@ public class GpasPseudonymGenerator implements Generator {
try {
responseEntity = retryTemplate.execute(
ctx -> restTemplate.exchange(gPasUrl, HttpMethod.POST, requestEntity,
String.class));
ctx -> restTemplate.exchange(gPasUrl, HttpMethod.POST, requestEntity,
String.class));
if (responseEntity.getStatusCode().is2xxSuccessful()) {
log.debug("API request succeeded. Response: {}", responseEntity.getStatusCode());
@ -119,16 +121,16 @@ public class GpasPseudonymGenerator implements Generator {
return responseEntity;
} catch (Exception unexpected) {
throw new PseudonymRequestFailed(
"API request due unexpected error unsuccessful gPas unsuccessful.", unexpected);
"API request due unexpected error unsuccessful gPas unsuccessful.", unexpected);
}
}
protected String getGpasRequestBody(String id) {
var requestParameters = new Parameters();
requestParameters.addParameter().setName("target")
.setValue(new StringType().setValue(psnTargetDomain));
.setValue(new StringType().setValue(psnTargetDomain));
requestParameters.addParameter().setName("original")
.setValue(new StringType().setValue(id));
.setValue(new StringType().setValue(id));
final IParser iParser = r4Context.newJsonParser();
return iParser.encodeResourceToString(requestParameters);
}

44
src/main/kotlin/dev/dnpm/etl/processor/config/AppConfigProperties.kt
View File

@ -66,6 +66,50 @@ data class GPasConfigProperties(
}
}
@ConfigurationProperties(GIcsConfigProperties.NAME)
data class GIcsConfigProperties(
/**
* Base URL to gICS System
*
*/
val gIcsBaseUri: String?,
val username: String?,
val password: String?,
/**
* If value is 'true' valid consent at processing time is mandatory for transmission of DNPM
* files otherwise they will be flagged and skipped.
* If value 'false' or missing consent status is assumed to be valid.
*/
val enabled: Boolean?,
/**
* gICS specific system
* **/
val personIdentifierSystem: String =
"https://ths-greifswald.de/fhir/gics/identifiers/Patienten-ID",
/**
* Domain of consent resources
* **/
val consentDomainName: String = "MII",
/**
* Value to expect in case of positiv consent
*/
val policyCode: String = "2.16.840.1.113883.3.1937.777.24.5.3.6",
/**
* Consent Policy which should be used for consent check
*/
val policySystem: String = "urn:oid:2.16.840.1.113883.3.1937.777.24.5.3",
val parameterVersion: String = "1.1"
) {
companion object {
const val NAME = "app.consent.gics"
}
}
@ConfigurationProperties(RestTargetProperties.NAME)
data class RestTargetProperties(
val uri: String?,

41
src/main/kotlin/dev/dnpm/etl/processor/config/AppConfiguration.kt
View File

@ -20,6 +20,9 @@
package dev.dnpm.etl.processor.config
import com.fasterxml.jackson.databind.ObjectMapper
import dev.dnpm.etl.processor.consent.ConsentCheckedIgnored
import dev.dnpm.etl.processor.consent.ICheckConsent
import dev.dnpm.etl.processor.consent.GicsConsentService
import dev.dnpm.etl.processor.monitoring.ConnectionCheckResult
import dev.dnpm.etl.processor.monitoring.ConnectionCheckService
import dev.dnpm.etl.processor.monitoring.GPasConnectionCheckService
@ -73,7 +76,8 @@ import kotlin.time.toJavaDuration
value = [
AppConfigProperties::class,
PseudonymizeConfigProperties::class,
GPasConfigProperties::class
GPasConfigProperties::class,
GIcsConfigProperties::class
]
)
@EnableScheduling
@ -86,22 +90,27 @@ class AppConfiguration {
return RestTemplate()
}
@Bean
fun appFhirConfig(): AppFhirConfig{
return AppFhirConfig()
}
@ConditionalOnProperty(value = ["app.pseudonymize.generator"], havingValue = "GPAS")
@Bean
fun gpasPseudonymGenerator(configProperties: GPasConfigProperties, retryTemplate: RetryTemplate, restTemplate: RestTemplate): Generator {
fun gpasPseudonymGenerator(configProperties: GPasConfigProperties, retryTemplate: RetryTemplate, restTemplate: RestTemplate, appFhirConfig: AppFhirConfig): Generator {
try {
if (!configProperties.sslCaLocation.isNullOrBlank()) {
return GpasPseudonymGenerator(
configProperties,
retryTemplate,
createCustomGpasRestTemplate(configProperties)
createCustomGpasRestTemplate(configProperties),appFhirConfig
)
}
} catch (e: Exception) {
throw RuntimeException(e)
}
return GpasPseudonymGenerator(configProperties, retryTemplate, restTemplate)
return GpasPseudonymGenerator(configProperties, retryTemplate, restTemplate,appFhirConfig)
}
@ConditionalOnProperty(value = ["app.pseudonymize.generator"], havingValue = "BUILDIN", matchIfMissing = true)
@ -113,20 +122,20 @@ class AppConfiguration {
@ConditionalOnProperty(value = ["app.pseudonymizer"], havingValue = "GPAS")
@ConditionalOnMissingBean
@Bean
fun gpasPseudonymGeneratorOnDeprecatedProperty(configProperties: GPasConfigProperties, retryTemplate: RetryTemplate, restTemplate: RestTemplate): Generator {
fun gpasPseudonymGeneratorOnDeprecatedProperty(configProperties: GPasConfigProperties, retryTemplate: RetryTemplate, restTemplate: RestTemplate, appFhirConfig: AppFhirConfig): Generator {
try {
if (!configProperties.sslCaLocation.isNullOrBlank()) {
return GpasPseudonymGenerator(
configProperties,
retryTemplate,
createCustomGpasRestTemplate(configProperties)
createCustomGpasRestTemplate(configProperties),appFhirConfig
)
}
} catch (e: Exception) {
throw RuntimeException(e)
}
return GpasPseudonymGenerator(configProperties, retryTemplate, restTemplate)
return GpasPseudonymGenerator(configProperties, retryTemplate, restTemplate,appFhirConfig)
}
private fun createCustomGpasRestTemplate(configProperties: GPasConfigProperties): RestTemplate {
@ -279,5 +288,23 @@ class AppConfiguration {
fun jdbcConfiguration(): AbstractJdbcConfiguration {
return AppJdbcConfiguration()
}
@Bean
@ConditionalOnMissingBean
fun constService(): ICheckConsent {
return ConsentCheckedIgnored()
}
@Bean
@ConditionalOnProperty(name = ["app.consent.gics.enabled"], havingValue = "true")
fun gicsAccessConsent( gIcsConfigProperties: GIcsConfigProperties,
retryTemplate: RetryTemplate, restTemplate: RestTemplate, appFhirConfig: AppFhirConfig): ICheckConsent {
return GicsConsentService(
gIcsConfigProperties,
retryTemplate,
restTemplate,
appFhirConfig
)
}
}

16
src/main/kotlin/dev/dnpm/etl/processor/config/AppFhirConfig.kt Normal file
View File

@ -0,0 +1,16 @@
package dev.dnpm.etl.processor.config
import ca.uhn.fhir.context.FhirContext
import org.springframework.context.annotation.Bean
import org.springframework.context.annotation.Configuration
@Configuration
class AppFhirConfig {
private val fhirCtx: FhirContext = FhirContext.forR4()
@Bean
fun fhirContext(): FhirContext {
return fhirCtx
}
}

11
src/main/kotlin/dev/dnpm/etl/processor/input/KafkaInputListener.kt
View File

@ -24,6 +24,7 @@ import de.ukw.ccc.bwhc.dto.Consent
import de.ukw.ccc.bwhc.dto.MtbFile
import dev.dnpm.etl.processor.PatientId
import dev.dnpm.etl.processor.RequestId
import dev.dnpm.etl.processor.consent.ConsentStatus
import dev.dnpm.etl.processor.services.RequestProcessor
import org.apache.kafka.clients.consumer.ConsumerRecord
import org.slf4j.LoggerFactory
@ -44,7 +45,7 @@ class KafkaInputListener(
} else {
RequestId("")
}
// fixme: add consent check
if (mtbFile.consent.status == Consent.Status.ACTIVE) {
logger.debug("Accepted MTB File for processing")
if (requestId.isBlank()) {
@ -55,9 +56,13 @@ class KafkaInputListener(
} else {
logger.debug("Accepted MTB File and process deletion")
if (requestId.isBlank()) {
requestProcessor.processDeletion(patientId)
requestProcessor.processDeletion(patientId, ConsentStatus.IGNORED)
} else {
requestProcessor.processDeletion(patientId, requestId)
requestProcessor.processDeletion(
patientId,
requestId,
ConsentStatus.IGNORED
)
}
}
}

23
src/main/kotlin/dev/dnpm/etl/processor/input/MtbFileRestController.kt
View File

@ -22,6 +22,8 @@ package dev.dnpm.etl.processor.input
import de.ukw.ccc.bwhc.dto.Consent
import de.ukw.ccc.bwhc.dto.MtbFile
import dev.dnpm.etl.processor.PatientId
import dev.dnpm.etl.processor.consent.ICheckConsent
import dev.dnpm.etl.processor.consent.ConsentStatus
import dev.dnpm.etl.processor.services.RequestProcessor
import org.slf4j.LoggerFactory
import org.springframework.http.ResponseEntity
@ -30,7 +32,7 @@ import org.springframework.web.bind.annotation.*
@RestController
@RequestMapping(path = ["mtbfile"])
class MtbFileRestController(
private val requestProcessor: RequestProcessor,
private val requestProcessor: RequestProcessor, private val constService: ICheckConsent
) {
private val logger = LoggerFactory.getLogger(MtbFileRestController::class.java)
@ -42,13 +44,24 @@ class MtbFileRestController(
@PostMapping
fun mtbFile(@RequestBody mtbFile: MtbFile): ResponseEntity<Void> {
if (mtbFile.consent.status == Consent.Status.ACTIVE) {
var consentStatus = constService.isConsented(mtbFile.patient.id)
if (mtbFile.consent.status == Consent.Status.ACTIVE && (consentStatus.equals(ConsentStatus.CONSENTED) || consentStatus.equals(
ConsentStatus.IGNORED
))
) {
logger.debug("Accepted MTB File for processing")
requestProcessor.processMtbFile(mtbFile)
} else {
logger.debug("Accepted MTB File and process deletion")
var msg = "Accepted MTB File and process deletion"
if (!consentStatus.equals(ConsentStatus.CONSENTED) || consentStatus.equals(ConsentStatus.IGNORED)) {
msg = "Accepted MTB File. But consent is missing, therefore process deletion."
}
if (mtbFile.consent.status == Consent.Status.REJECTED) consentStatus =
ConsentStatus.CONSENT_REJECTED
logger.debug(msg)
val patientId = PatientId(mtbFile.patient.id)
requestProcessor.processDeletion(patientId)
requestProcessor.processDeletion(patientId, consentStatus)
}
return ResponseEntity.accepted().build()
}
@ -56,7 +69,7 @@ class MtbFileRestController(
@DeleteMapping(path = ["{patientId}"])
fun deleteData(@PathVariable patientId: String): ResponseEntity<Void> {
logger.debug("Accepted patient ID to process deletion")
requestProcessor.processDeletion(PatientId(patientId))
requestProcessor.processDeletion(PatientId(patientId), ConsentStatus.IGNORED)
return ResponseEntity.accepted().build()
}

3
src/main/kotlin/dev/dnpm/etl/processor/monitoring/RequestStatus.kt
View File

@ -24,5 +24,6 @@ enum class RequestStatus(val value: String) {
WARNING("warning"),
ERROR("error"),
UNKNOWN("unknown"),
DUPLICATION("duplication")
DUPLICATION("duplication"),
CONSENTMISSING("no-consent")
}

28
src/main/kotlin/dev/dnpm/etl/processor/services/RequestProcessor.kt
View File

@ -23,6 +23,7 @@ import com.fasterxml.jackson.databind.ObjectMapper
import de.ukw.ccc.bwhc.dto.MtbFile
import dev.dnpm.etl.processor.*
import dev.dnpm.etl.processor.config.AppConfigProperties
import dev.dnpm.etl.processor.consent.ConsentStatus
import dev.dnpm.etl.processor.monitoring.Report
import dev.dnpm.etl.processor.monitoring.Request
import dev.dnpm.etl.processor.monitoring.RequestStatus
@ -59,7 +60,8 @@ class RequestProcessor(
mtbFile pseudonymizeWith pseudonymizeService
mtbFile anonymizeContentWith pseudonymizeService
val request = MtbFileSender.MtbFileRequest(requestId, transformationService.transform(mtbFile))
val request =
MtbFileSender.MtbFileRequest(requestId, transformationService.transform(mtbFile))
val patientPseudonym = PatientPseudonym(request.mtbFile.patient.id)
@ -105,21 +107,29 @@ class RequestProcessor(
val lastMtbFileRequestForPatient =
requestService.lastMtbFileRequestForPatientPseudonym(patientPseudonym)
val isLastRequestDeletion = requestService.isLastRequestWithKnownStatusDeletion(patientPseudonym)
val isLastRequestDeletion =
requestService.isLastRequestWithKnownStatusDeletion(patientPseudonym)
return null != lastMtbFileRequestForPatient
&& !isLastRequestDeletion
&& lastMtbFileRequestForPatient.fingerprint == fingerprint(pseudonymizedMtbFile)
}
fun processDeletion(patientId: PatientId) {
processDeletion(patientId, randomRequestId())
fun processDeletion(patientId: PatientId, isConsented: ConsentStatus) {
processDeletion(patientId, randomRequestId(), isConsented)
}
fun processDeletion(patientId: PatientId, requestId: RequestId) {
fun processDeletion(patientId: PatientId, requestId: RequestId, isConsented: ConsentStatus) {
try {
val patientPseudonym = pseudonymizeService.patientPseudonym(patientId)
val requestStatus: RequestStatus = when (isConsented) {
ConsentStatus.CONSENT_MISSING -> RequestStatus.CONSENTMISSING
ConsentStatus.FAILED_TO_ASK -> RequestStatus.ERROR
ConsentStatus.CONSENTED, ConsentStatus.IGNORED,
ConsentStatus.CONSENT_REJECTED -> RequestStatus.UNKNOWN
}
requestService.save(
Request(
requestId,
@ -127,11 +137,14 @@ class RequestProcessor(
patientId,
fingerprint(patientPseudonym.value),
RequestType.DELETE,
RequestStatus.UNKNOWN
requestStatus
)
)
val responseStatus = sender.send(MtbFileSender.DeleteRequest(requestId, patientPseudonym))
val responseStatus =
sender.send(MtbFileSender.DeleteRequest(requestId, patientPseudonym))
//fixme: publish proper report if consent check failed
applicationEventPublisher.publishEvent(
ResponseEvent(
@ -171,5 +184,4 @@ class RequestProcessor(
.lowercase()
)
}
}

14
src/test/java/dev/dnpm/etl/processor/consent/GicsConsentServiceTest.java Normal file
View File

@ -0,0 +1,14 @@
package dev.dnpm.etl.processor.consent;
import static org.assertj.core.api.Assertions.assertThat;
import org.junit.jupiter.api.Test;
//@ExtendWith(MockitoExtension.class)
public class GicsConsentServiceTest {
@Test
void isConsented() {
}
}

83
src/test/kotlin/dev/dnpm/etl/processor/input/KafkaInputListenerTest.kt
View File

@ -24,6 +24,7 @@ import de.ukw.ccc.bwhc.dto.Consent
import de.ukw.ccc.bwhc.dto.MtbFile
import de.ukw.ccc.bwhc.dto.Patient
import dev.dnpm.etl.processor.anyValueClass
import dev.dnpm.etl.processor.consent.ConsentStatus
import dev.dnpm.etl.processor.services.RequestProcessor
import org.apache.kafka.clients.consumer.ConsumerRecord
import org.apache.kafka.common.header.internals.RecordHeader
@ -35,6 +36,7 @@ import org.junit.jupiter.api.extension.ExtendWith
import org.mockito.Mock
import org.mockito.junit.jupiter.MockitoExtension
import org.mockito.kotlin.any
import org.mockito.kotlin.eq
import org.mockito.kotlin.times
import org.mockito.kotlin.verify
import java.util.*
@ -48,7 +50,7 @@ class KafkaInputListenerTest {
@BeforeEach
fun setup(
@Mock requestProcessor: RequestProcessor
@Mock requestProcessor: RequestProcessor,
) {
this.requestProcessor = requestProcessor
this.objectMapper = ObjectMapper()
@ -63,7 +65,15 @@ class KafkaInputListenerTest {
.withConsent(Consent.builder().withStatus(Consent.Status.ACTIVE).build())
.build()
kafkaInputListener.onMessage(ConsumerRecord("testtopic", 0, 0, "", this.objectMapper.writeValueAsString(mtbFile)))
kafkaInputListener.onMessage(
ConsumerRecord(
"testtopic",
0,
0,
"",
this.objectMapper.writeValueAsString(mtbFile)
)
)
verify(requestProcessor, times(1)).processMtbFile(any())
}
@ -75,9 +85,20 @@ class KafkaInputListenerTest {
.withConsent(Consent.builder().withStatus(Consent.Status.REJECTED).build())
.build()
kafkaInputListener.onMessage(ConsumerRecord("testtopic", 0, 0, "", this.objectMapper.writeValueAsString(mtbFile)))
kafkaInputListener.onMessage(
ConsumerRecord(
"testtopic",
0,
0,
"",
this.objectMapper.writeValueAsString(mtbFile)
)
)
verify(requestProcessor, times(1)).processDeletion(anyValueClass())
verify(requestProcessor, times(1)).processDeletion(
anyValueClass(),
eq(ConsentStatus.IGNORED)
)
}
@Test
@ -87,9 +108,28 @@ class KafkaInputListenerTest {
.withConsent(Consent.builder().withStatus(Consent.Status.ACTIVE).build())
.build()
val headers = RecordHeaders(listOf(RecordHeader("requestId", UUID.randomUUID().toString().toByteArray())))
val headers = RecordHeaders(
listOf(
RecordHeader(
"requestId",
UUID.randomUUID().toString().toByteArray()
)
)
)
kafkaInputListener.onMessage(
ConsumerRecord("testtopic", 0, 0, -1L, TimestampType.NO_TIMESTAMP_TYPE, -1, -1, "", this.objectMapper.writeValueAsString(mtbFile), headers, Optional.empty())
ConsumerRecord(
"testtopic",
0,
0,
-1L,
TimestampType.NO_TIMESTAMP_TYPE,
-1,
-1,
"",
this.objectMapper.writeValueAsString(mtbFile),
headers,
Optional.empty()
)
)
verify(requestProcessor, times(1)).processMtbFile(any(), anyValueClass())
@ -102,11 +142,34 @@ class KafkaInputListenerTest {
.withConsent(Consent.builder().withStatus(Consent.Status.REJECTED).build())
.build()
val headers = RecordHeaders(listOf(RecordHeader("requestId", UUID.randomUUID().toString().toByteArray())))
kafkaInputListener.onMessage(
ConsumerRecord("testtopic", 0, 0, -1L, TimestampType.NO_TIMESTAMP_TYPE, -1, -1, "", this.objectMapper.writeValueAsString(mtbFile), headers, Optional.empty())
val headers = RecordHeaders(
listOf(
RecordHeader(
"requestId",
UUID.randomUUID().toString().toByteArray()
)
)
)
kafkaInputListener.onMessage(
ConsumerRecord(
"testtopic",
0,
0,
-1L,
TimestampType.NO_TIMESTAMP_TYPE,
-1,
-1,
"",
this.objectMapper.writeValueAsString(mtbFile),
headers,
Optional.empty()
)
)
verify(requestProcessor, times(1)).processDeletion(
anyValueClass(),
anyValueClass(),
eq(ConsentStatus.IGNORED)
)
verify(requestProcessor, times(1)).processDeletion(anyValueClass(), anyValueClass())
}
}

20
src/test/kotlin/dev/dnpm/etl/processor/input/MtbFileRestControllerTest.kt
View File

@ -22,13 +22,14 @@ package dev.dnpm.etl.processor.input
import com.fasterxml.jackson.databind.ObjectMapper
import de.ukw.ccc.bwhc.dto.*
import dev.dnpm.etl.processor.anyValueClass
import dev.dnpm.etl.processor.consent.ConsentCheckedIgnored
import dev.dnpm.etl.processor.consent.ConsentStatus
import dev.dnpm.etl.processor.services.RequestProcessor
import org.junit.jupiter.api.BeforeEach
import org.junit.jupiter.api.Test
import org.junit.jupiter.api.extension.ExtendWith
import org.mockito.Mock
import org.mockito.Mockito.times
import org.mockito.Mockito.verify
import org.mockito.Mockito.*
import org.mockito.junit.jupiter.MockitoExtension
import org.mockito.kotlin.any
import org.springframework.http.MediaType
@ -51,7 +52,10 @@ class MtbFileRestControllerTest {
@Mock requestProcessor: RequestProcessor
) {
this.requestProcessor = requestProcessor
val controller = MtbFileRestController(requestProcessor)
val controller = MtbFileRestController(
requestProcessor, ConsentCheckedIgnored()
)
this.mockMvc = MockMvcBuilders.standaloneSetup(controller).build()
}
@ -128,7 +132,10 @@ class MtbFileRestControllerTest {
}
}
verify(requestProcessor, times(1)).processDeletion(anyValueClass())
verify(requestProcessor, times(1)).processDeletion(
anyValueClass(),
org.mockito.kotlin.eq(ConsentStatus.CONSENT_REJECTED)
)
}
@Test
@ -139,7 +146,10 @@ class MtbFileRestControllerTest {
}
}
verify(requestProcessor, times(1)).processDeletion(anyValueClass())
verify(requestProcessor, times(1)).processDeletion(
anyValueClass(),
org.mockito.kotlin.eq(ConsentStatus.IGNORED)
)
}
}

9
src/test/kotlin/dev/dnpm/etl/processor/services/RequestProcessorTest.kt
View File

@ -23,6 +23,7 @@ import com.fasterxml.jackson.databind.ObjectMapper
import de.ukw.ccc.bwhc.dto.*
import dev.dnpm.etl.processor.*
import dev.dnpm.etl.processor.config.AppConfigProperties
import dev.dnpm.etl.processor.consent.ConsentStatus
import dev.dnpm.etl.processor.monitoring.Request
import dev.dnpm.etl.processor.monitoring.RequestStatus
import dev.dnpm.etl.processor.monitoring.RequestType
@ -337,7 +338,7 @@ class RequestProcessorTest {
MtbFileSender.Response(status = RequestStatus.UNKNOWN)
}.whenever(sender).send(any<MtbFileSender.DeleteRequest>())
this.requestProcessor.processDeletion(TEST_PATIENT_ID)
this.requestProcessor.processDeletion(TEST_PATIENT_ID, isConsented = ConsentStatus.IGNORED)
val requestCaptor = argumentCaptor<Request>()
verify(requestService, times(1)).save(requestCaptor.capture())
@ -355,7 +356,7 @@ class RequestProcessorTest {
MtbFileSender.Response(status = RequestStatus.SUCCESS)
}.whenever(sender).send(any<MtbFileSender.DeleteRequest>())
this.requestProcessor.processDeletion(TEST_PATIENT_ID)
this.requestProcessor.processDeletion(TEST_PATIENT_ID, isConsented = ConsentStatus.IGNORED)
val eventCaptor = argumentCaptor<ResponseEvent>()
verify(applicationEventPublisher, times(1)).publishEvent(eventCaptor.capture())
@ -373,7 +374,7 @@ class RequestProcessorTest {
MtbFileSender.Response(status = RequestStatus.ERROR)
}.whenever(sender).send(any<MtbFileSender.DeleteRequest>())
this.requestProcessor.processDeletion(TEST_PATIENT_ID)
this.requestProcessor.processDeletion(TEST_PATIENT_ID, isConsented = ConsentStatus.IGNORED)
val eventCaptor = argumentCaptor<ResponseEvent>()
verify(applicationEventPublisher, times(1)).publishEvent(eventCaptor.capture())
@ -385,7 +386,7 @@ class RequestProcessorTest {
fun testShouldSendDeleteRequestWithPseudonymErrorAndSaveErrorRequestStatus() {
doThrow(RuntimeException()).whenever(pseudonymizeService).patientPseudonym(anyValueClass())
this.requestProcessor.processDeletion(TEST_PATIENT_ID)
this.requestProcessor.processDeletion(TEST_PATIENT_ID, isConsented = ConsentStatus.IGNORED)
val requestCaptor = argumentCaptor<Request>()
verify(requestService, times(1)).save(requestCaptor.capture())