feat: auto import files

build.gradle.kts

@@ -32,6 +32,7 @@ dependencies {
     implementation("org.jetbrains.kotlin:kotlin-reflect")
     implementation("org.flywaydb:flyway-database-postgresql")
     implementation("org.flywaydb:flyway-mysql")
+    implementation("org.springframework.integration:spring-integration-file")
     developmentOnly("org.springframework.boot:spring-boot-devtools")
     developmentOnly("org.springframework.boot:spring-boot-docker-compose")
     runtimeOnly("org.mariadb.jdbc:mariadb-java-client")

src/main/kotlin/dev/pcvolkmer/onco/grzmetadataprocessor/config/AppIntegrationConfig.kt

@@ -0,0 +1,97 @@
+package dev.pcvolkmer.onco.grzmetadataprocessor.config
+
+import dev.pcvolkmer.onco.grzmetadataprocessor.data.File
+import dev.pcvolkmer.onco.grzmetadataprocessor.data.FileRepository
+import dev.pcvolkmer.onco.grzmetadataprocessor.data.FileType
+import org.apache.tomcat.util.buf.HexUtils
+import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty
+import org.springframework.boot.context.properties.ConfigurationProperties
+import org.springframework.boot.context.properties.EnableConfigurationProperties
+import org.springframework.context.annotation.Bean
+import org.springframework.context.annotation.Configuration
+import org.springframework.integration.dsl.IntegrationFlow
+import org.springframework.integration.dsl.Pollers
+import org.springframework.integration.file.dsl.Files
+import org.springframework.util.Assert
+import java.nio.file.Path
+import java.security.DigestInputStream
+import java.security.MessageDigest
+import java.time.Duration
+import kotlin.io.path.*
+import kotlin.time.Duration.Companion.minutes
+import kotlin.time.toJavaDuration
+
+@ConfigurationProperties(AppSourceFsProperties.NAME)
+data class AppSourceFsProperties(
+    val directory: Path? = null,
+    val pollDelay: Duration = 1.minutes.toJavaDuration(),
+) {
+    companion object {
+        const val NAME = "app.source.fs"
+    }
+}
+
+@Configuration
+@EnableConfigurationProperties(AppSourceFsProperties::class)
+class AppIntegrationConfig {
+
+    @Bean
+    @ConditionalOnProperty(
+        name = ["app.source.fs.directory"]
+    )
+    fun fileInputFlow(
+        applicationFsProperties: AppSourceFsProperties,
+        fileRepository: FileRepository
+    ): IntegrationFlow {
+        val sourceDirectory = applicationFsProperties.directory
+        Assert.state(null != sourceDirectory && sourceDirectory.isDirectory()) {
+            "Property 'app.source.fs.active' is 'true' but source directory is not available"
+        }
+        return IntegrationFlow
+            .from(
+                Files.inboundAdapter(sourceDirectory!!.toFile()).useWatchService(true)
+            )
+            .log()
+            .handle { msg ->
+                val path = Path(msg.payload.toString())
+                val relativePath = applicationFsProperties.directory.relativize(Path(msg.payload.toString())).pathString
+                fileRepository.findByFilePath(relativePath).ifPresentOrElse({
+                    // File already present
+                }, {
+                    fileRepository.save(
+                        File(
+                            filePath = relativePath,
+                            labDataId = null,
+                            fileChecksum = calcFileChecksum(path),
+                            fileSizeInBytes = path.fileSize(),
+                            fileType = getFileType(path),
+                        )
+                    )
+                })
+
+            }
+            .get()
+    }
+
+    private fun calcFileChecksum(path: Path): String {
+        val messageDigest = MessageDigest.getInstance("SHA-256")
+        val digestInputStream = DigestInputStream(path.inputStream(), messageDigest)
+        digestInputStream.readAllBytes()
+        return HexUtils.toHexString(messageDigest.digest())
+    }
+
+    private fun getFileType(path: Path): FileType? {
+        return if (path.toString().lowercase().endsWith(".fastq.gz")) {
+            FileType.FASTQ
+        } else if (path.toString().lowercase().endsWith(".bed")) {
+            FileType.BED
+        } else if (path.toString().lowercase().endsWith(".bam")) {
+            FileType.BAM
+        } else if (path.toString().lowercase().endsWith(".vcf")) {
+            FileType.VCF
+        } else {
+            null
+        }
+    }
+
+}

src/main/kotlin/dev/pcvolkmer/onco/grzmetadataprocessor/data/File.kt

@@ -17,8 +17,8 @@ data class File(
     val labDataId: Long?,
     val filePath: String? = null,
     val fileType: FileType? = null,
-    var fileChecksum: String? = null,
+    var fileChecksum: String = "",
-    var fileSizeInBytes: Long? = null,
+    var fileSizeInBytes: Long = 0,
 ) {
     fun calcFileChecksum(): String {
         if (filePath == null) {
@@ -26,7 +26,6 @@ data class File(
         }
         val path = Path.of(filePath)
         val messageDigest = MessageDigest.getInstance("SHA-256")
-
         val digestInputStream = DigestInputStream(path.inputStream(), messageDigest)
         digestInputStream.readAllBytes()
         return HexUtils.toHexString(messageDigest.digest())
@@ -50,4 +49,5 @@ enum class FileType(val value: String) {
 interface FileRepository : CrudRepository<File, Long> {
     fun findByLabDataId(labDataId: Long): MutableList<File>
     fun findByLabDataIdIsNull(): List<File>
+    fun findByFilePath(filePath: String): Optional<File>
 }

src/main/resources/db/migrations/mariadb/V0_1_0_2__FileChecksums.sql

@@ -0,0 +1 @@
+ALTER TABLE tbl_file DROP INDEX file_checksum;

src/main/resources/db/migrations/postgresql/V0_1_0_2__FileChecksums.sql

@@ -0,0 +1 @@
+ALTER TABLE tbl_file DROP CONSTRAINT tbl_file_file_checksum_key;

src/main/resources/templates/unusedfiles.html

@@ -40,7 +40,7 @@
                         <label class="optional">
                             Zugehörige Probe (Einsendenummer)
                             <select name="labDataId">
-                                <option>Keine Zuordnung</option>
+                                <option value="">Keine Zuordnung</option>
                                 <option th:each="labData : ${labDatas}" th:selected="${file.labDataId == labData.id}" th:value="${labData.id}" th:text="${labData.einsendenummer}"></option>
                             </select>
                         </label>
@@ -50,10 +50,10 @@
                         <label>
                             Dateityp
                             <select name="fileType">
-                                <option th:selected="${file.fileType.value == 'bam'}">BAM</option>
+                                <option th:selected="${file.fileType != null and file.fileType.value == 'bam'}" value="BAM">BAM</option>
-                                <option th:selected="${file.fileType.value == 'vcf'}">Vcf</option>
+                                <option th:selected="${file.fileType != null and file.fileType.value == 'vcf'}" value="VCF">Vcf</option>
-                                <option th:selected="${file.fileType.value == 'bed'}">BED</option>
+                                <option th:selected="${file.fileType != null and file.fileType.value == 'bed'}" value="BED">BED</option>
-                                <option th:selected="${file.fileType.value == 'fastq'}">FASTQ</option>
+                                <option th:selected="${file.fileType != null and file.fileType.value == 'fastq'}" value="FASTQ" >FASTQ</option>
                             </select>
                         </label>
                     </div>