chore: update dependencies

This will replace previous used filter and only includes oBDS messages without
information about tumor board information.

Cargo.toml

@@ -11,11 +11,11 @@ clap = { version = "4.5", features = ["std", "help", "usage", "derive", "error-c
 console = "0.15"
 csv = "1.3"
 dialoguer = "0.11"
-itertools = "0.13"
+itertools = "0.14"
 mysql = "25.0"
 serde = { version = "1.0", features = ["derive"] }
 urlencoding = "2.1"
-regex = "1.10"
+regex = "1.11"
 
 [profile.release]
 opt-level = "s"

README.md

@@ -2,6 +2,9 @@
 
 Anwendung zur Durchführung einer Plausibilitätsprüfung anhand der Daten für die BZKF Real World Data Platform.
 
+**Hinweis:** Dies ist eine Weiterführung des ursprünglichen
+Projekts [bzkf-rwdp-check](https://github.com/CCC-MF/bzkf-rwdp-check)
+
 ## Aufbau der ETL-Strecke an den Standorten
 
 Die Daten werden aus der Onkostar-Datenbank ausgelesen und in Apache-Kafka eingespeist.
@@ -110,4 +113,8 @@ Mithilfe dieser Anwendung kann auch der aktuelle Inhalt der Datenbank gegen die
 Export verglichen werden.
 
 Der Befehl `check-export` kann zusammen mit der Angabe der Protokolldatei (`--file`) und der Angabe des
-Exports (`--export-package=...`) und den Optionen für den Datenbankzugriff ausgeführt werden.
+Exports (`--package=...` bzw. `--export-package=...`) und den Optionen für den Datenbankzugriff ausgeführt werden.
+
+Zur Überprüfung werden irrelevante Leerzeichen und Zeilenumbrüche entfernt und _Self-Closed-Tags_ ersetzt
+(`<Meldeanlass />` => `<Meldeanlass></Meldeanlass>`),
+da in Onkostar in der Datenbank und der LKR-Export-Protokolldatei verschiedene Formatierungen verwendet werden (können).

src/cli.rs

@@ -1,7 +1,7 @@
 /*
  * This file is part of bzkf-rwdp-check
  *
- * Copyright (C) 2024 Comprehensive Cancer Center Mainfranken and contributors.
+ * Copyright (C) 2024 the original author or authors.
  *
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of the GNU General Public License as published by
@@ -62,6 +62,8 @@ pub enum SubCommand {
         year: String,
         #[arg(long, value_parser = value_is_date, help = "Ignoriere LKR-Exporte seit Datum")]
         ignore_exports_since: Option<String>,
+        #[arg(long, help = "Ignoriere Meldungen, die nicht im oBDS 2.x Format sind")]
+        ignore_non_obds_2: bool,
         #[arg(long, help = "Meldungen mit externer Diagnose einschließen")]
         include_extern: bool,
         #[arg(
@@ -103,6 +105,8 @@ pub enum SubCommand {
         year: String,
         #[arg(long, value_parser = value_is_date, help = "Ignoriere LKR-Exporte seit Datum")]
         ignore_exports_since: Option<String>,
+        #[arg(long, help = "Ignoriere Meldungen, die nicht im oBDS 2.x Format sind")]
+        ignore_non_obds_2: bool,
         #[arg(long, help = "Export mit Trennzeichen ';' für Excel")]
         xls_csv: bool,
         #[arg(long, help = "Meldungen mit externer Diagnose einschließen")]
@@ -142,6 +146,8 @@ pub enum SubCommand {
         year: String,
         #[arg(long, value_parser = value_is_date, help = "Ignoriere LKR-Exporte seit Datum")]
         ignore_exports_since: Option<String>,
+        #[arg(long, help = "Ignoriere Meldungen, die nicht im oBDS 2.x Format sind")]
+        ignore_non_obds_2: bool,
         #[arg(long, help = "Meldungen mit externer Diagnose einschließen")]
         include_extern: bool,
         #[arg(
@@ -173,8 +179,13 @@ pub enum SubCommand {
         user: String,
         #[arg(short, long, help = "LKR-Export-Protokoll-Datei")]
         file: PathBuf,
-        #[arg(long, help = "Exportpaketnummer", default_value = "0")]
+        #[arg(
-        export_package: u16,
+            long,
+            alias = "export-package",
+            help = "Exportpaketnummer",
+            default_value = "0"
+        )]
+        package: u16,
     },
 }
 

src/common.rs

@@ -1,7 +1,7 @@
 /*
  * This file is part of bzkf-rwdp-check
  *
- * Copyright (C) 2024 Comprehensive Cancer Center Mainfranken and contributors.
+ * Copyright (C) 2024 the original author or authors.
  *
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of the GNU General Public License as published by

src/database.rs

@@ -1,7 +1,7 @@
 /*
  * This file is part of bzkf-rwdp-check
  *
- * Copyright (C) 2024 Comprehensive Cancer Center Mainfranken and contributors.
+ * Copyright (C) 2024 the original author or authors.
  *
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of the GNU General Public License as published by
@@ -55,6 +55,7 @@ impl DatabaseSource {
         &self,
         year: &str,
         ignore_exports_since: &str,
+        ignore_non_obds_2: bool,
         include_extern: bool,
         include_histo_zyto: bool,
         schema_versions: bool,
@@ -62,6 +63,7 @@ impl DatabaseSource {
         let params = params! {
             "year" => year,
             "ignore_exports_since" => ignore_exports_since,
+            "ignore_non_obds_2" => if ignore_non_obds_2 { 1 } else { 0 },
             "include_extern" => if include_extern { 1 } else { 0 },
             "include_histo_zyto" => if include_histo_zyto { 1 } else { 0 }
         };
@@ -97,6 +99,7 @@ impl DatabaseSource {
         &self,
         year: &str,
         ignore_exports_since: &str,
+        ignore_non_obds_2: bool,
         use_pat_id: bool,
         include_extern: bool,
         include_histo_zyto: bool,
@@ -109,6 +112,7 @@ impl DatabaseSource {
                         params! {
                             "year" => year,
                             "ignore_exports_since" => ignore_exports_since,
+                            "ignore_non_obds_2" => if ignore_non_obds_2 { 1 } else { 0 },
                             "include_extern" => if include_extern { 1 } else { 0 },
                             "include_histo_zyto" => if include_histo_zyto { 1 } else { 0 }
                         },
@@ -134,14 +138,14 @@ impl DatabaseSource {
         Err(())
     }
 
-    pub fn exported(&self, export_id: u16) -> Result<Vec<(String, String)>, ()> {
+    pub fn exported(&self, package: u16) -> Result<Vec<(String, String)>, ()> {
         match Pool::new(self.0.as_str()) {
             Ok(pool) => {
                 if let Ok(mut connection) = pool.try_get_conn(Duration::from_secs(3)) {
                     return match connection.exec_map(
                         EXPORTED_TO_LKR,
                         params! {
-                            "export_id" => export_id,
+                            "export_id" => package,
                         },
                         |(id, xml_data)| (id, xml_data),
                     ) {

src/lkrexport.rs

@@ -1,7 +1,7 @@
 /*
  * This file is part of bzkf-rwdp-check
  *
- * Copyright (C) 2024 Comprehensive Cancer Center Mainfranken and contributors.
+ * Copyright (C) 2024 the original author or authors.
  *
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of the GNU General Public License as published by

src/main.rs

@@ -1,7 +1,7 @@
 /*
  * This file is part of bzkf-rwdp-check
  *
- * Copyright (C) 2024 Comprehensive Cancer Center Mainfranken and contributors.
+ * Copyright (C) 2024 the original author or authors.
  *
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of the GNU General Public License as published by
@@ -51,9 +51,9 @@ fn request_password_if_none(password: Option<String>) -> String {
     }
 }
 
-fn sanitize_year(year: String) -> String {
+fn sanitize_year(year: &str) -> String {
     if year.len() == 4 {
-        year
+        year.to_string()
     } else {
         format!("2{:0>3}", year)
     }
@@ -129,12 +129,13 @@ fn main() -> Result<(), Box<dyn Error>> {
             user,
             year,
             ignore_exports_since,
+            ignore_non_obds_2,
             include_extern,
             include_histo_zyto,
             schema_versions,
         } => {
             let password = request_password_if_none(password);
-            let year = sanitize_year(year);
+            let year = sanitize_year(&year);
 
             let _ = term.write_line(
                 &style(format!("Warte auf Daten für das Diagnosejahr {}...", year))
@@ -148,6 +149,7 @@ fn main() -> Result<(), Box<dyn Error>> {
                 .check(
                     &year,
                     &ignore_exports_since.unwrap_or("9999-12-31".into()),
+                    ignore_non_obds_2,
                     include_extern,
                     include_histo_zyto,
                     schema_versions,
@@ -169,12 +171,13 @@ fn main() -> Result<(), Box<dyn Error>> {
             output,
             year,
             ignore_exports_since,
+            ignore_non_obds_2,
             xls_csv,
             include_extern,
             include_histo_zyto,
         } => {
             let password = request_password_if_none(password);
-            let year = sanitize_year(year);
+            let year = sanitize_year(&year);
 
             let _ = term.write_line(
                 &style(format!("Warte auf Daten für das Diagnosejahr {}...", year))
@@ -188,6 +191,7 @@ fn main() -> Result<(), Box<dyn Error>> {
                 .export(
                     &year,
                     &ignore_exports_since.unwrap_or("9999-12-31".into()),
+                    ignore_non_obds_2,
                     pat_id,
                     include_extern,
                     include_histo_zyto,
@@ -232,11 +236,12 @@ fn main() -> Result<(), Box<dyn Error>> {
             file,
             year,
             ignore_exports_since,
+            ignore_non_obds_2,
             include_extern,
             include_histo_zyto,
         } => {
             let password = request_password_if_none(password);
-            let year = sanitize_year(year);
+            let year = sanitize_year(&year);
 
             let _ = term.write_line(
                 &style(format!("Warte auf Daten für das Diagnosejahr {}...", year))
@@ -250,6 +255,7 @@ fn main() -> Result<(), Box<dyn Error>> {
                 .export(
                     &year,
                     &ignore_exports_since.unwrap_or("9999-12-31".into()),
+                    ignore_non_obds_2,
                     pat_id,
                     include_extern,
                     include_histo_zyto,
@@ -263,7 +269,7 @@ fn main() -> Result<(), Box<dyn Error>> {
 
             let mut not_in_csv = db_items
                 .iter()
-                .filter(|db_item| {
+                .filter(|&db_item| {
                     !csv_items
                         .iter()
                         .map(|csv_item| &csv_item.condition_id)
@@ -293,7 +299,7 @@ fn main() -> Result<(), Box<dyn Error>> {
 
             not_in_csv
                 .iter()
-                .for_each(|item| match Check::is_relevant(&item.icd_10_code) {
+                .for_each(|&item| match Check::is_relevant(&item.icd_10_code) {
                     true => {
                         let _ = term.write_line(&format!(
                             "{:<64}   {:<10}   {:<5}   {:<5}   {}",
@@ -324,7 +330,7 @@ fn main() -> Result<(), Box<dyn Error>> {
 
             let mut not_in_db = csv_items
                 .iter()
-                .filter(|csv_item| {
+                .filter(|&csv_item| {
                     !db_items
                         .iter()
                         .map(|db_item| &db_item.condition_id)
@@ -444,14 +450,14 @@ fn main() -> Result<(), Box<dyn Error>> {
             port,
             user,
             file,
-            export_package,
+            package,
         } => {
             let password = request_password_if_none(password);
 
             let _ = term.write_line(
                 &style(format!(
                     "Warte auf Daten für den LKR-Export '{}'...",
-                    export_package
+                    package
                 ))
                 .blue()
                 .bright()
@@ -461,7 +467,7 @@ fn main() -> Result<(), Box<dyn Error>> {
             let db = DatabaseSource::new(&database, &host, &password, port, &user);
 
             let db_entries = db
-                .exported(export_package)
+                .exported(package)
                 .map_err(|_e| "Fehler bei Zugriff auf die Datenbank")?;
 
             let db_meldungen = db_entries
@@ -498,6 +504,16 @@ fn main() -> Result<(), Box<dyn Error>> {
                 .to_string(),
             );
 
+            fn print_missing_ids(missing_ids: &[&String], term: &Term) {
+                missing_ids.iter().sorted().for_each(|&item| {
+                    let _ = term.write_line(&format!(
+                        "{} ({})",
+                        item,
+                        to_database_id(item).unwrap_or("?".into())
+                    ));
+                });
+            }
+
             if db_meldungen.len() != xml_meldungen.len() {
                 let _ = term.write_line(
                     &style("\nNicht übereinstimmende Anzahl an Meldungen:")
@@ -522,13 +538,7 @@ fn main() -> Result<(), Box<dyn Error>> {
                             .to_string(),
                     );
 
-                    missing_db_ids.iter().sorted().for_each(|&item| {
+                    print_missing_ids(&missing_db_ids, &term);
-                        let _ = term.write_line(&format!(
-                            "{} ({})",
-                            item,
-                            to_database_id(item).unwrap_or("?".into())
-                        ));
-                    });
                 }
 
                 if !missing_xml_ids.is_empty() {
@@ -538,13 +548,7 @@ fn main() -> Result<(), Box<dyn Error>> {
                             .to_string(),
                     );
 
-                    missing_xml_ids.iter().sorted().for_each(|&item| {
+                    print_missing_ids(&missing_xml_ids, &term);
-                        let _ = term.write_line(&format!(
-                            "{} ({})",
-                            item,
-                            to_database_id(item).unwrap_or("?".into())
-                        ));
-                    });
                 }
             }
 
@@ -568,7 +572,7 @@ fn main() -> Result<(), Box<dyn Error>> {
                         .to_string(),
                 );
 
-                multiple_meldung_entries.iter().for_each(|item| {
+                multiple_meldung_entries.iter().for_each(|&item| {
                     let _ = term.write_line(&item.to_string());
                 });
             }
@@ -604,7 +608,7 @@ fn main() -> Result<(), Box<dyn Error>> {
 
                 different_content
                     .iter()
-                    .sorted_by(|id1, id2| {
+                    .sorted_by(|&id1, &id2| {
                         to_database_id(id1)
                             .unwrap_or_default()
                             .cmp(&to_database_id(id2).unwrap_or_default())

src/opal.rs

@@ -1,7 +1,7 @@
 /*
  * This file is part of bzkf-rwdp-check
  *
- * Copyright (C) 2024 Comprehensive Cancer Center Mainfranken and contributors.
+ * Copyright (C) 2024 the original author or authors.
  *
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of the GNU General Public License as published by

src/resources/export.sql

@@ -1,7 +1,7 @@
 /*
  * This file is part of bzkf-rwdp-check
  *
- * Copyright (C) 2024 Comprehensive Cancer Center Mainfranken and contributors.
+ * Copyright (C) 2024 the original author or authors.
  *
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of the GNU General Public License as published by
@@ -29,15 +29,17 @@ FROM (
         EXTRACTVALUE(lme.xml_daten, '//Patienten_Stammdaten/@Patient_ID') AS pid,
         lme.versionsnummer,
         SHA2(CONCAT('https://fhir.diz.uk-erlangen.de/identifiers/onkostar-xml-condition-id|', EXTRACTVALUE(lme.xml_daten, '//Patienten_Stammdaten/@Patient_ID'), 'condition', EXTRACTVALUE(lme.xml_daten, '//Diagnose/@Tumor_ID')), 256) AS cond_id,
-        SUBSTRING_INDEX(EXTRACTVALUE(lm.xml_daten, '//Primaertumor_ICD_Code'), ' ', 1) AS condcodingcode,
+        SUBSTRING_INDEX(EXTRACTVALUE(lme.xml_daten, '//Primaertumor_ICD_Code'), ' ', 1) AS condcodingcode,
-        SUBSTRING_INDEX(EXTRACTVALUE(lm.xml_daten, '//Diagnosedatum'), ' ', 1) AS diagnosedatum,
+        SUBSTRING_INDEX(EXTRACTVALUE(lme.xml_daten, '//Diagnosedatum'), ' ', 1) AS diagnosedatum,
-        SUBSTRING_INDEX(SUBSTRING_INDEX(EXTRACTVALUE(lm.xml_daten, '//Diagnosedatum'), ' ', 1), '.', -1) AS diagnosejahr
+        SUBSTRING_INDEX(SUBSTRING_INDEX(EXTRACTVALUE(lme.xml_daten, '//Diagnosedatum'), ' ', 1), '.', -1) AS diagnosejahr
     FROM lkr_meldung_export lme
-    JOIN lkr_meldung lm ON (lm.id = lme.lkr_meldung AND lme.typ <> '-1' AND lm.extern <= :include_extern)
+    WHERE lme.xml_daten LIKE '%ICD_Version%'
-    WHERE lm.xml_daten LIKE '%ICD_Version%'
+        AND lme.typ <> -1
-        AND SUBSTRING_INDEX(SUBSTRING_INDEX(EXTRACTVALUE(lm.xml_daten, '//Diagnosedatum'), ' ', 1), '.', -1) = :year
+        AND lme.xml_daten NOT LIKE '%<Menge_Tumorkonferenz%'
-        AND (lm.xml_daten LIKE '%<cTNM%' OR lm.xml_daten LIKE '%<pTNM%' OR lm.xml_daten LIKE '%<Menge_Histologie>%' OR lm.xml_daten LIKE '%<Menge_Weitere_Klassifikation>%')
+        AND SUBSTRING_INDEX(SUBSTRING_INDEX(EXTRACTVALUE(lme.xml_daten, '//Diagnosedatum'), ' ', 1), '.', -1) = :year
-        AND (lm.xml_daten NOT LIKE '%histologie_zytologie%' OR 1 = :include_histo_zyto)
+        AND (lme.xml_daten NOT LIKE '%histologie_zytologie%' OR 1 = :include_histo_zyto)
+        AND (EXTRACTVALUE(lme.xml_daten, '//Meldende_Stelle') NOT LIKE '%9999%' OR 1 <= :include_extern)
+        AND (EXTRACTVALUE(lme.xml_daten, '//ADT_GEKID/@Schema_Version') LIKE '2.%' OR 1 = :ignore_non_obds_2)
     ) o1
     LEFT OUTER JOIN (
 

src/resources/exported-to-lkr.sql

@@ -1,7 +1,7 @@
 /*
  * This file is part of bzkf-rwdp-check
  *
- * Copyright (C) 2024 Comprehensive Cancer Center Mainfranken and contributors.
+ * Copyright (C) 2024 the original author or authors.
  *
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of the GNU General Public License as published by

src/resources/mod.rs

@@ -1,7 +1,7 @@
 /*
  * This file is part of bzkf-rwdp-check
  *
- * Copyright (C) 2024 Comprehensive Cancer Center Mainfranken and contributors.
+ * Copyright (C) 2024 the original author or authors.
  *
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of the GNU General Public License as published by

src/resources/query.sql

@@ -1,7 +1,7 @@
 /*
  * This file is part of bzkf-rwdp-check
  *
- * Copyright (C) 2024 Comprehensive Cancer Center Mainfranken and contributors.
+ * Copyright (C) 2024 the original author or authors.
  *
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of the GNU General Public License as published by
@@ -121,10 +121,11 @@ FROM (
     FROM lkr_meldung_export lme
     WHERE lme.xml_daten LIKE '%ICD_Version%'
         AND lme.typ <> -1
+        AND lme.xml_daten NOT LIKE '%<Menge_Tumorkonferenz%'
         AND SUBSTRING_INDEX(SUBSTRING_INDEX(EXTRACTVALUE(lme.xml_daten, '//Diagnosedatum'), ' ', 1), '.', -1) = :year
-        AND (lme.xml_daten LIKE '%<cTNM%' OR lme.xml_daten LIKE '%<pTNM%' OR lme.xml_daten LIKE '%<Menge_Histologie>%' OR lme.xml_daten LIKE '%<Menge_Weitere_Klassifikation>%')
         AND (lme.xml_daten NOT LIKE '%histologie_zytologie%' OR 1 = :include_histo_zyto)
         AND (EXTRACTVALUE(lme.xml_daten, '//Meldende_Stelle') NOT LIKE '%9999%' OR 1 <= :include_extern)
+        AND (EXTRACTVALUE(lme.xml_daten, '//ADT_GEKID/@Schema_Version') LIKE '2.%' OR 1 = :ignore_non_obds_2)
  ) o1
 LEFT OUTER JOIN (
     SELECT DISTINCT

src/resources/query_with_schema_version.sql

@@ -1,7 +1,7 @@
 /*
  * This file is part of bzkf-rwdp-check
  *
- * Copyright (C) 2024 Comprehensive Cancer Center Mainfranken and contributors.
+ * Copyright (C) 2024 the original author or authors.
  *
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of the GNU General Public License as published by
@@ -121,10 +121,11 @@ FROM (
     FROM lkr_meldung_export lme
     WHERE lme.xml_daten LIKE '%ICD_Version%'
         AND lme.typ <> -1
+        AND lme.xml_daten NOT LIKE '%<Menge_Tumorkonferenz%'
         AND SUBSTRING_INDEX(SUBSTRING_INDEX(EXTRACTVALUE(lme.xml_daten, '//Diagnosedatum'), ' ', 1), '.', -1) = :year
-        AND (lme.xml_daten LIKE '%<cTNM%' OR lme.xml_daten LIKE '%<pTNM%' OR lme.xml_daten LIKE '%<Menge_Histologie>%' OR lme.xml_daten LIKE '%<Menge_Weitere_Klassifikation>%')
         AND (lme.xml_daten NOT LIKE '%histologie_zytologie%' OR 1 = :include_histo_zyto)
         AND (EXTRACTVALUE(lme.xml_daten, '//Meldende_Stelle') NOT LIKE '%9999%' OR 1 <= :include_extern)
+        AND (EXTRACTVALUE(lme.xml_daten, '//ADT_GEKID/@Schema_Version') LIKE '2.%' OR 1 = :ignore_non_obds_2)
  ) o1
 LEFT OUTER JOIN (
     SELECT DISTINCT