From 76428c27ef263677814a132c9a3791934fc03132 Mon Sep 17 00:00:00 2001 From: Philipp Schmelter Date: Tue, 17 Dec 2024 14:02:54 +0100 Subject: [PATCH 1/5] Review --- .github/dependabot.yml | 3 ++- CHANGELOG.md | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 65035fa7..0c7f085c 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -9,9 +9,10 @@ updates: target-branch: main reviewers: - t-ober - - sensarmad + - jo-bao - sebastian-peter - danielfeismann + - staudtMarius ignore: - dependency-name: org.jgrapht:jgrapht-core versions: diff --git a/CHANGELOG.md b/CHANGELOG.md index 6cfdf8ff..3efbe90e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,7 +5,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ## [Unreleased] - +### Added +- Added Bao and Staudt to list of reviewers [#277](https://github.com/ie3-institute/simBench2psdm/issues/277) ## [1.0.0] - 2021-08-03 ### Added - Basic functionality to convert SimBench data sets to [PowerSystemDataModel](https://github.com/ie3-institute/powersystemdatamodel) From 897f9debda72b9eb90c484ca4275b71d0987b239 Mon Sep 17 00:00:00 2001 From: Philipp Schmelter Date: Tue, 17 Dec 2024 18:24:17 +0100 Subject: [PATCH 2/5] debug --- src/test/scala/edu/ie3/simbench/io/ExtractorSpec.scala | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/test/scala/edu/ie3/simbench/io/ExtractorSpec.scala b/src/test/scala/edu/ie3/simbench/io/ExtractorSpec.scala index d592d7a1..1d6c8f63 100644 --- a/src/test/scala/edu/ie3/simbench/io/ExtractorSpec.scala +++ b/src/test/scala/edu/ie3/simbench/io/ExtractorSpec.scala @@ -80,9 +80,7 @@ class ExtractorSpec extends UnitSpec with IoUtils with Matchers { "return a correct map of codes to UUIDs" in { extractor.download() val uuidMap = extractor.extractUUIDMap() - uuidMap should contain key "1-LV-urban6--0-sw" uuidMap should contain key "1-LV-rural1--2-no_sw" - uuidMap("1-LV-urban6--0-sw") should fullyMatch regex uuidPattern uuidMap("1-LV-rural1--2-no_sw") should fullyMatch regex uuidPattern } From 0ee5222cf6998103be4ac949d8d1a2a467675154 Mon Sep 17 00:00:00 2001 From: Philipp Schmelter Date: Mon, 10 Feb 2025 13:06:49 +0100 Subject: [PATCH 3/5] Redesigning Test --- .../edu/ie3/simbench/io/ExtractorSpec.scala | 150 ++++++++++-------- 1 file changed, 83 insertions(+), 67 deletions(-) diff --git a/src/test/scala/edu/ie3/simbench/io/ExtractorSpec.scala b/src/test/scala/edu/ie3/simbench/io/ExtractorSpec.scala index 1d6c8f63..dca1f74a 100644 --- a/src/test/scala/edu/ie3/simbench/io/ExtractorSpec.scala +++ b/src/test/scala/edu/ie3/simbench/io/ExtractorSpec.scala @@ -4,20 +4,67 @@ import edu.ie3.simbench.config.SimbenchConfig import edu.ie3.simbench.config.SimbenchConfig.Io import edu.ie3.simbench.config.SimbenchConfig.Io.{Input, Output} import edu.ie3.simbench.config.SimbenchConfig.Io.Input.Download - -import java.io.{File, PrintWriter} -import org.scalatest.matchers.should.Matchers import edu.ie3.test.common.UnitSpec +import org.scalatest.{BeforeAndAfterAll, BeforeAndAfterEach} +import org.scalatest.matchers.should.Matchers + +import java.nio.file.{Files, Path} +import scala.util.Try import scala.util.matching.Regex -class ExtractorSpec extends UnitSpec with IoUtils with Matchers { - // Define configurations for the extractor - val download: Download = Download( +class ExtractorSpec + extends UnitSpec + with IoUtils + with Matchers + with BeforeAndAfterAll + with BeforeAndAfterEach { + + private var tempDir: Path = _ + + override def beforeAll(): Unit = { + super.beforeAll() + tempDir = Files.createTempDirectory("extractorTest") + } + + override def afterAll(): Unit = { + Try(deleteRecursively(tempDir)) + super.afterAll() + } + + override def beforeEach(): Unit = { + super.beforeEach() + } + + override def afterEach(): Unit = { + super.afterEach() + } + + private def deleteRecursively(path: Path): Unit = { + val file = path.toFile + if (file.exists()) { + if (file.isDirectory) { + file.listFiles().foreach(child => deleteRecursively(child.toPath)) + } + file.delete() + } + } + + private def createTestFile(fileName: String, content: String): Path = { + val filePath = tempDir.resolve(fileName) + Files.write(filePath, content.getBytes("UTF-8")) + filePath + } + + /* ------------------------------------------------------------ + * Example config for Extractor + * ------------------------------------------------------------ */ + private val download: Download = Download( baseUrl = "https://daks.uni-kassel.de/bitstreams", failOnExistingFiles = false, - directory = "testData/download" // Valid download location + directory = tempDir.toString ) - val input: Input = Input( + + private val input: Input = Input( csv = SimbenchConfig.CsvConfig( directoryHierarchy = false, fileEncoding = "UTF-8", @@ -26,7 +73,8 @@ class ExtractorSpec extends UnitSpec with IoUtils with Matchers { ), download = download ) - val output: Output = Output( + + private val output: Output = Output( compress = true, csv = SimbenchConfig.CsvConfig( directoryHierarchy = false, @@ -36,95 +84,63 @@ class ExtractorSpec extends UnitSpec with IoUtils with Matchers { ), targetDir = "convertedData" ) - val io: Io = Io(input = input, output = output, simbenchCodes = List.empty) - val simbenchConfig: SimbenchConfig = SimbenchConfig( + + private val io: Io = Io( + input = input, + output = output, + simbenchCodes = List.empty + ) + + private val simbenchConfig: SimbenchConfig = SimbenchConfig( conversion = SimbenchConfig.Conversion(removeSwitches = false), io = io ) - val extractor = new Extractor(simbenchConfig) - val uuidPattern: Regex = - "[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}".r + private val extractor = new Extractor(simbenchConfig) - // Utility to create test files - private def createTestFile( - directory: String, - fileName: String, - content: String - ): File = { - val dir = new File(directory) - if (!dir.exists()) dir.mkdirs() - val file = new File(s"$directory/$fileName") - val writer = new PrintWriter(file) - writer.write(content) - writer.close() - file - } + private val uuidPattern: Regex = + "[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}".r - // Utility to clean up test files - private def deleteDir(directory: String): Unit = { - val dir = new File(directory) - if (dir.exists()) dir.listFiles().foreach(_.delete()) - dir.delete() - } + /* ------------------------------------------------------------ + * Test Cases + * ------------------------------------------------------------ */ "The extractor" should { + "ensure that the download correctly downloads and saves the expected data file" in { extractor.download() - val downloadedFile = new File( - s"${simbenchConfig.io.input.download.directory}/simbench_datalinks.csv" - ) + val downloadedFile = tempDir.resolve("simbench_datalinks.csv").toFile downloadedFile.exists() shouldBe true } "return a correct map of codes to UUIDs" in { extractor.download() + val uuidMap = extractor.extractUUIDMap() + uuidMap("1-LV-urban6--0-sw") should fullyMatch regex uuidPattern uuidMap should contain key "1-LV-rural1--2-no_sw" uuidMap("1-LV-rural1--2-no_sw") should fullyMatch regex uuidPattern } - "check that an appropriate exception (IllegalArgumentException) is thrown when attempting to extract data that does not contain the required columns" in { - val invalidDir = "testData/invalidStructure" + "throw an IllegalArgumentException when CSV does not contain the required columns" in { createTestFile( - invalidDir, "simbench_datalinks.csv", "Invalid,Content\nOnly,OneColumn" ) - val invalidConfig = simbenchConfig.copy( - io = simbenchConfig.io.copy( - input = simbenchConfig.io.input.copy( - download = simbenchConfig.io.input.download.copy( - directory = invalidDir - ) - ) - ) - ) - val invalidExtractor = new Extractor(invalidConfig) - an[IllegalArgumentException] should be thrownBy invalidExtractor - .extractUUIDMap() - deleteDir(invalidDir) + + an[IllegalArgumentException] should be thrownBy { + extractor.extractUUIDMap() + } } + "handle cases with no valid UUIDs" in { - val invalidUUIDPath = "testData/invalidUUIDs" createTestFile( - invalidUUIDPath, "simbench_datalinks.csv", "code,csv\n1-LV-urban6--0-sw,invalid_uuid\n1-LV-rural1--2-no_sw,another_invalid_uuid" ) - val invalidUUIDConfig = simbenchConfig.copy( - io = simbenchConfig.io.copy( - input = simbenchConfig.io.input.copy( - download = simbenchConfig.io.input.download.copy( - directory = invalidUUIDPath - ) - ) - ) - ) - val invalidUUIDExtractor = new Extractor(invalidUUIDConfig) - val uuidMap = invalidUUIDExtractor.extractUUIDMap() + + val uuidMap = extractor.extractUUIDMap() uuidMap shouldBe empty - deleteDir(invalidUUIDPath) } } } From 7ff588e685b3f9e1ad3c6536ce971c11e6e4aa21 Mon Sep 17 00:00:00 2001 From: Philipp Schmelter Date: Mon, 10 Feb 2025 13:15:06 +0100 Subject: [PATCH 4/5] Redesigning Test --- .../edu/ie3/simbench/io/ExtractorSpec.scala | 112 ++++++++---------- 1 file changed, 52 insertions(+), 60 deletions(-) diff --git a/src/test/scala/edu/ie3/simbench/io/ExtractorSpec.scala b/src/test/scala/edu/ie3/simbench/io/ExtractorSpec.scala index dca1f74a..84e903ac 100644 --- a/src/test/scala/edu/ie3/simbench/io/ExtractorSpec.scala +++ b/src/test/scala/edu/ie3/simbench/io/ExtractorSpec.scala @@ -5,15 +5,15 @@ import edu.ie3.simbench.config.SimbenchConfig.Io import edu.ie3.simbench.config.SimbenchConfig.Io.{Input, Output} import edu.ie3.simbench.config.SimbenchConfig.Io.Input.Download import edu.ie3.test.common.UnitSpec -import org.scalatest.{BeforeAndAfterAll, BeforeAndAfterEach} import org.scalatest.matchers.should.Matchers +import org.scalatest.{BeforeAndAfterAll, BeforeAndAfterEach} import java.nio.file.{Files, Path} import scala.util.Try import scala.util.matching.Regex class ExtractorSpec - extends UnitSpec + extends UnitSpec with IoUtils with Matchers with BeforeAndAfterAll @@ -21,14 +21,51 @@ class ExtractorSpec private var tempDir: Path = _ + private var extractor: Extractor = _ + private var uuidPattern: Regex = _ + override def beforeAll(): Unit = { super.beforeAll() + tempDir = Files.createTempDirectory("extractorTest") - } - override def afterAll(): Unit = { - Try(deleteRecursively(tempDir)) - super.afterAll() + val download: Download = Download( + baseUrl = "https://daks.uni-kassel.de/bitstreams", + failOnExistingFiles = false, + directory = tempDir.toString + ) + + val input: Input = Input( + csv = SimbenchConfig.CsvConfig( + directoryHierarchy = false, + fileEncoding = "UTF-8", + fileEnding = ".csv", + separator = ";" + ), + download = download + ) + + val output: Output = Output( + compress = true, + csv = SimbenchConfig.CsvConfig( + directoryHierarchy = false, + fileEncoding = "UTF-8", + fileEnding = ".csv", + separator = ";" + ), + targetDir = "convertedData" + ) + + val io: Io = Io(input = input, output = output, simbenchCodes = List.empty) + + val simbenchConfig: SimbenchConfig = SimbenchConfig( + conversion = SimbenchConfig.Conversion(removeSwitches = false), + io = io + ) + + extractor = new Extractor(simbenchConfig) + + uuidPattern = "[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}".r } override def beforeEach(): Unit = { @@ -39,6 +76,11 @@ class ExtractorSpec super.afterEach() } + override def afterAll(): Unit = { + Try(deleteRecursively(tempDir)) + super.afterAll() + } + private def deleteRecursively(path: Path): Unit = { val file = path.toFile if (file.exists()) { @@ -55,55 +97,9 @@ class ExtractorSpec filePath } - /* ------------------------------------------------------------ - * Example config for Extractor - * ------------------------------------------------------------ */ - private val download: Download = Download( - baseUrl = "https://daks.uni-kassel.de/bitstreams", - failOnExistingFiles = false, - directory = tempDir.toString - ) - - private val input: Input = Input( - csv = SimbenchConfig.CsvConfig( - directoryHierarchy = false, - fileEncoding = "UTF-8", - fileEnding = ".csv", - separator = ";" - ), - download = download - ) - - private val output: Output = Output( - compress = true, - csv = SimbenchConfig.CsvConfig( - directoryHierarchy = false, - fileEncoding = "UTF-8", - fileEnding = ".csv", - separator = ";" - ), - targetDir = "convertedData" - ) - - private val io: Io = Io( - input = input, - output = output, - simbenchCodes = List.empty - ) - - private val simbenchConfig: SimbenchConfig = SimbenchConfig( - conversion = SimbenchConfig.Conversion(removeSwitches = false), - io = io - ) - - private val extractor = new Extractor(simbenchConfig) - - private val uuidPattern: Regex = - "[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}".r - - /* ------------------------------------------------------------ - * Test Cases - * ------------------------------------------------------------ */ + /* ------------------------------------------------------------------ + * Testcases + * ------------------------------------------------------------------ */ "The extractor" should { @@ -115,7 +111,6 @@ class ExtractorSpec "return a correct map of codes to UUIDs" in { extractor.download() - val uuidMap = extractor.extractUUIDMap() uuidMap("1-LV-urban6--0-sw") should fullyMatch regex uuidPattern uuidMap should contain key "1-LV-rural1--2-no_sw" @@ -123,10 +118,7 @@ class ExtractorSpec } "throw an IllegalArgumentException when CSV does not contain the required columns" in { - createTestFile( - "simbench_datalinks.csv", - "Invalid,Content\nOnly,OneColumn" - ) + createTestFile("simbench_datalinks.csv", "Invalid,Content\nOnly,OneColumn") an[IllegalArgumentException] should be thrownBy { extractor.extractUUIDMap() From 8ccd20cdb5cebc1bcd5fe4443d486ccbf820ea1f Mon Sep 17 00:00:00 2001 From: Philipp Schmelter Date: Mon, 10 Feb 2025 13:15:32 +0100 Subject: [PATCH 5/5] spotless --- src/test/scala/edu/ie3/simbench/io/ExtractorSpec.scala | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/src/test/scala/edu/ie3/simbench/io/ExtractorSpec.scala b/src/test/scala/edu/ie3/simbench/io/ExtractorSpec.scala index 84e903ac..58f1a1a3 100644 --- a/src/test/scala/edu/ie3/simbench/io/ExtractorSpec.scala +++ b/src/test/scala/edu/ie3/simbench/io/ExtractorSpec.scala @@ -13,7 +13,7 @@ import scala.util.Try import scala.util.matching.Regex class ExtractorSpec - extends UnitSpec + extends UnitSpec with IoUtils with Matchers with BeforeAndAfterAll @@ -65,7 +65,8 @@ class ExtractorSpec extractor = new Extractor(simbenchConfig) - uuidPattern = "[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}".r + uuidPattern = + "[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}".r } override def beforeEach(): Unit = { @@ -118,7 +119,10 @@ class ExtractorSpec } "throw an IllegalArgumentException when CSV does not contain the required columns" in { - createTestFile("simbench_datalinks.csv", "Invalid,Content\nOnly,OneColumn") + createTestFile( + "simbench_datalinks.csv", + "Invalid,Content\nOnly,OneColumn" + ) an[IllegalArgumentException] should be thrownBy { extractor.extractUUIDMap()