From 8591df9dbd07ee8b571dfd3d972f229624b8af46 Mon Sep 17 00:00:00 2001 From: Andrei Molnar <1995am@pm.me> Date: Thu, 19 Nov 2026 09:58:48 +0100 Subject: [PATCH] Migration system + separate properties migration. --- .../molnarandrei/sopa/data/yaml/Config.java | 2 + .../molnarandrei/sopa/data/yaml/Project.java | 4 +- .../com/molnarandrei/sopa/data/yaml/Yaml.java | 14 ++++ .../sopa/migrations/MigrateUtils.java | 36 ++++++++ .../sopa/migrations/Migration.java | 84 +++++++++++++++++++ .../sopa/project/ProjectRepository.java | 6 +- .../sopa/project/ProjectService.java | 9 +- .../sopa/properties/PropertiesRepository.java | 21 +++-- src/main/resources/AGENTS.md | 3 + .../sopa/migrations/MigrationTest.java | 68 +++++++++++++++ .../sopa/project/ProjectFolderTest.java | 2 +- 11 files changed, 227 insertions(+), 22 deletions(-) create mode 100644 src/main/java/com/molnarandrei/sopa/migrations/MigrateUtils.java create mode 100644 src/main/java/com/molnarandrei/sopa/migrations/Migration.java create mode 100644 src/test/java/com/molnarandrei/sopa/migrations/MigrationTest.java diff --git a/src/main/java/com/molnarandrei/sopa/data/yaml/Config.java b/src/main/java/com/molnarandrei/sopa/data/yaml/Config.java index b52e643..e15745a 100644 --- a/src/main/java/com/molnarandrei/sopa/data/yaml/Config.java +++ b/src/main/java/com/molnarandrei/sopa/data/yaml/Config.java @@ -1,6 +1,7 @@ package com.molnarandrei.sopa.data.yaml; +import com.molnarandrei.sopa.migrations.Migration; import com.molnarandrei.sopa.data.ProjectMonitors; import com.molnarandrei.sopa.data.RepositoryException; import com.molnarandrei.sopa.events.Events; @@ -130,6 +131,7 @@ public class Config { // So: iterate a copy, and only start listening once the saved projects are loaded. for (var project : List.copyOf(loadedProjects)) { try { + Migration.migrate(project); ProjectMonitors.start(project.getParent()); } catch (RepositoryException logAndIgnore) { log.log(Level.WARNING, "Cannot load previously loaded project at %s.".formatted(project), logAndIgnore); diff --git a/src/main/java/com/molnarandrei/sopa/data/yaml/Project.java b/src/main/java/com/molnarandrei/sopa/data/yaml/Project.java index 6b51b66..4698663 100644 --- a/src/main/java/com/molnarandrei/sopa/data/yaml/Project.java +++ b/src/main/java/com/molnarandrei/sopa/data/yaml/Project.java @@ -1,5 +1,6 @@ package com.molnarandrei.sopa.data.yaml; +import com.molnarandrei.sopa.migrations.Migration; import com.molnarandrei.sopa.soap.SoapVersion; import java.util.HashMap; @@ -9,7 +10,8 @@ import java.util.Map; public class Project { public static final String HEADER = "# com.molnarandrei.sopa.Project"; - private String version = "1.0.2"; + /** Of the files of the project: {@link Migration} brings older projects to the current one. */ + private String version = String.valueOf(Migration.VERSION); private String name = ""; /** By url, in the order they were added. */ private Map endpoints = new LinkedHashMap<>(); diff --git a/src/main/java/com/molnarandrei/sopa/data/yaml/Yaml.java b/src/main/java/com/molnarandrei/sopa/data/yaml/Yaml.java index ff32730..1cf1d98 100644 --- a/src/main/java/com/molnarandrei/sopa/data/yaml/Yaml.java +++ b/src/main/java/com/molnarandrei/sopa/data/yaml/Yaml.java @@ -3,6 +3,9 @@ package com.molnarandrei.sopa.data.yaml; import tools.jackson.core.JacksonException; import tools.jackson.dataformat.yaml.YAMLMapper; +import java.util.LinkedHashMap; +import java.util.Map; + public class Yaml { private static final YAMLMapper mapper = HumanComfyYamlMapper.get(); @@ -62,6 +65,17 @@ public class Yaml { return Config.HEADER + '\n' + mapper.writeValueAsString(config); } + /** Any yml as plain maps, lists and scalars, as migrations read the files of older versions. */ + @SuppressWarnings("unchecked") + public static Map toMap(String value) throws JacksonException { + var map = mapper.readValue(value, LinkedHashMap.class); + return map == null ? new LinkedHashMap<>() : map; + } + + public static String toYaml(String header, Map map) { + return header + '\n' + mapper.writeValueAsString(map); + } + /** Whether both would be written the same, whatever the order of their maps. */ public static boolean sameContent(Object a, Object b) { return mapper.valueToTree(a).equals(mapper.valueToTree(b)); diff --git a/src/main/java/com/molnarandrei/sopa/migrations/MigrateUtils.java b/src/main/java/com/molnarandrei/sopa/migrations/MigrateUtils.java new file mode 100644 index 0000000..ac16f17 --- /dev/null +++ b/src/main/java/com/molnarandrei/sopa/migrations/MigrateUtils.java @@ -0,0 +1,36 @@ +package com.molnarandrei.sopa.migrations; + +import com.molnarandrei.sopa.data.FileNames; +import com.molnarandrei.sopa.data.yaml.Yaml; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.List; +import java.util.Map; + +/** File helpers for migrations, which read the files of older versions as plain maps rather than as the app objects. */ +public class MigrateUtils { + + /** The testcase folders in {@code projectFolder}, in no particular order. */ + public static List testCaseFolders(Path projectFolder) throws IOException { + try (var children = Files.list(projectFolder)) { + return children.filter(folder -> Files.isDirectory(folder) && FileNames.hasTestCaseFolderName(folder)).toList(); + } + } + + /** The step folders in {@code testCaseFolder}, in no particular order. */ + public static List stepFolders(Path testCaseFolder) throws IOException { + try (var children = Files.list(testCaseFolder)) { + return children.filter(FileNames::isStepFolder).toList(); + } + } + + public static Map readYaml(Path ymlPath) throws IOException { + return Yaml.toMap(Files.readString(ymlPath)); + } + + public static void writeYaml(Path ymlPath, String header, Map yaml) throws IOException { + Files.writeString(ymlPath, Yaml.toYaml(header, yaml)); + } +} diff --git a/src/main/java/com/molnarandrei/sopa/migrations/Migration.java b/src/main/java/com/molnarandrei/sopa/migrations/Migration.java new file mode 100644 index 0000000..5f39cf9 --- /dev/null +++ b/src/main/java/com/molnarandrei/sopa/migrations/Migration.java @@ -0,0 +1,84 @@ +package com.molnarandrei.sopa.migrations; + +import com.molnarandrei.sopa.data.FileSystemCache; +import com.molnarandrei.sopa.data.RepositoryException; +import com.molnarandrei.sopa.data.yaml.Project; +import com.molnarandrei.sopa.data.yaml.TestCase; +import com.molnarandrei.sopa.project.ProjectRepository; +import com.molnarandrei.sopa.properties.PropertiesRepository; +import tools.jackson.core.JacksonException; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.Map; +import java.util.logging.Logger; + +/** + * Brings the files of a project up to the {@link #VERSION} of this app, one version at a time, before anything else + * reads them. Migration {@code n} takes a project from version {@code n} to {@code n + 1}. + */ +public class Migration { + private static final Logger log = Logger.getLogger(Migration.class.getName()); + + /** The version of the projects of this app. */ + public static final int VERSION = 2; + + private Migration() {} + + /** Migrates the project of {@code projectPath}, its project.yml, when its version is older. Overwrites its AGENTS.md. */ + public static void migrate(Path projectPath) throws RepositoryException { + try { + var version = versionOf(MigrateUtils.readYaml(projectPath)); + if (version >= VERSION) return; + + var projectFolder = projectPath.getParent(); + log.info("Migrating project %s from version %d to %d.".formatted(projectPath, version, VERSION)); + for (; version < VERSION; version++) + switch (version) { + case 1 -> migration_1(projectFolder); + default -> throw new IllegalStateException("No migration from version %d".formatted(version)); + } + + var project = MigrateUtils.readYaml(projectPath); + project.put("version", String.valueOf(VERSION)); + MigrateUtils.writeYaml(projectPath, Project.HEADER, project); + ProjectRepository.writeAgentsFile(projectFolder); + FileSystemCache.remove(projectFolder); + } catch (IOException | JacksonException cannotMigrate) { + throw new RepositoryException(projectPath, "Cannot migrate project", cannotMigrate); + } + } + + /** The whole number before any dot: "1.0.2" is 1. A project without version is 1. */ + static int versionOf(Map project) { + var version = project.get("version"); + if (version == null) return 1; + return Integer.parseInt(String.valueOf(version).split("\\.")[0].strip()); + } + + /** Properties move from the yml of the project and its testcases to their .properties files. */ + static void migration_1(Path projectFolder) throws IOException, RepositoryException { + var ymlPaths = new ArrayList(); + ymlPaths.add(projectFolder.resolve("project.yml")); + for (var testCaseFolder : MigrateUtils.testCaseFolders(projectFolder)) + ymlPaths.add(testCaseFolder.resolve("testcase.yml")); + + for (var ymlPath : ymlPaths) { + if (! Files.exists(ymlPath)) continue; + var yaml = MigrateUtils.readYaml(ymlPath); + if (! (yaml.remove("properties") instanceof Map moved)) continue; + log.info("Moving testcase properties to .properties file %s".formatted(ymlPath)); + + var properties = new HashMap(); + moved.forEach((key, value) -> properties.put(String.valueOf(key), value == null ? "" : String.valueOf(value))); + // A properties file already there was written by a newer version: its values win. + var propertiesPath = PropertiesRepository.of(ymlPath); + properties.putAll(PropertiesRepository.get(propertiesPath)); + PropertiesRepository.store(propertiesPath, properties); + MigrateUtils.writeYaml(ymlPath, ymlPath.endsWith("project.yml") ? Project.HEADER : TestCase.HEADER, yaml); + } + } +} diff --git a/src/main/java/com/molnarandrei/sopa/project/ProjectRepository.java b/src/main/java/com/molnarandrei/sopa/project/ProjectRepository.java index b3d0f20..19eb5b0 100644 --- a/src/main/java/com/molnarandrei/sopa/project/ProjectRepository.java +++ b/src/main/java/com/molnarandrei/sopa/project/ProjectRepository.java @@ -16,10 +16,9 @@ import tools.jackson.core.JacksonException; import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; +import java.nio.file.StandardCopyOption; import java.util.ArrayList; -import java.util.Comparator; import java.util.List; -import java.util.logging.Logger; public class ProjectRepository { @@ -115,12 +114,13 @@ public class ProjectRepository { } } + /** Writes the AGENTS.md of this version of the app, over any there is. */ public static void writeAgentsFile(Path projectDir) throws RepositoryException { var agentsFile = projectDir.resolve("AGENTS.md"); try (var template = ProjectRepository.class.getResourceAsStream("/AGENTS.md")) { if (template == null) throw new IOException("AGENTS.md is not packaged"); - Files.copy(template, agentsFile); + Files.copy(template, agentsFile, StandardCopyOption.REPLACE_EXISTING); } catch (IOException io) { throw new RepositoryException(agentsFile, "Could not create agents file", io); } diff --git a/src/main/java/com/molnarandrei/sopa/project/ProjectService.java b/src/main/java/com/molnarandrei/sopa/project/ProjectService.java index 5877e71..069a15f 100644 --- a/src/main/java/com/molnarandrei/sopa/project/ProjectService.java +++ b/src/main/java/com/molnarandrei/sopa/project/ProjectService.java @@ -2,25 +2,20 @@ package com.molnarandrei.sopa.project; import java.nio.file.Files; import com.molnarandrei.sopa.data.yaml.Kind; +import com.molnarandrei.sopa.migrations.Migration; import com.molnarandrei.sopa.undo.Undo; import com.molnarandrei.sopa.Utils; import com.molnarandrei.sopa.data.ProjectMonitors; import com.molnarandrei.sopa.data.yaml.Project; -import com.molnarandrei.sopa.data.yaml.Step; -import com.molnarandrei.sopa.data.yaml.TestCase; -import com.molnarandrei.sopa.project.ProjectRepository; import com.molnarandrei.sopa.data.RepositoryException; import com.molnarandrei.sopa.events.Events; import com.molnarandrei.sopa.events.ProjectEvent; -import com.molnarandrei.sopa.step.StepRepository; -import com.molnarandrei.sopa.testcase.TestCaseRepository; import java.nio.file.Path; import java.util.logging.Level; import java.util.logging.Logger; import static com.molnarandrei.sopa.Utils.renameIndexed; -import static com.molnarandrei.sopa.data.yaml.Kind.project; /** * All operations over projects at the application level belong here. @@ -181,8 +176,10 @@ public class ProjectService { return true; } + /** Migrates the project to this version of the app, before anything reads it, and opens it. */ public static void open(Path projectPath) { try { + Migration.migrate(projectPath); ProjectRepository.get(projectPath); ProjectMonitors.start(projectPath.getParent()); } catch (RepositoryException repo) { diff --git a/src/main/java/com/molnarandrei/sopa/properties/PropertiesRepository.java b/src/main/java/com/molnarandrei/sopa/properties/PropertiesRepository.java index 5aae0ea..6d8522f 100644 --- a/src/main/java/com/molnarandrei/sopa/properties/PropertiesRepository.java +++ b/src/main/java/com/molnarandrei/sopa/properties/PropertiesRepository.java @@ -1,16 +1,10 @@ package com.molnarandrei.sopa.properties; -import com.molnarandrei.sopa.data.ProjectMonitors; -import com.molnarandrei.sopa.data.RepositoryException; +import com.molnarandrei.sopa.data.*; -import java.io.IOException; -import java.io.StringReader; -import java.io.StringWriter; -import java.nio.file.Files; -import java.nio.file.Path; -import java.util.HashMap; -import java.util.Map; -import java.util.Properties; +import java.io.*; +import java.nio.file.*; +import java.util.*; /** * The properties of a project or testcase, in a java .properties file next to its yml: @@ -46,13 +40,18 @@ public class PropertiesRepository { /** Writes {@code properties}, unless the file already holds them. */ public static void update(Path propertiesPath, Map properties) throws RepositoryException { if (get(propertiesPath).equals(properties)) return; + store(propertiesPath, properties); + ProjectMonitors.trigger(projectDirectoryOf(propertiesPath)); + } + + /** Writes {@code properties} without notifying the monitor: for projects not open yet, as when migrating. */ + public static void store(Path propertiesPath, Map properties) throws RepositoryException { try { var stored = new Properties(); stored.putAll(properties); var text = new StringWriter(); stored.store(text, null); Files.writeString(propertiesPath, text.toString()); - ProjectMonitors.trigger(projectDirectoryOf(propertiesPath)); } catch (IOException io) { throw new RepositoryException(propertiesPath, "Cannot write properties", io); } diff --git a/src/main/resources/AGENTS.md b/src/main/resources/AGENTS.md index 66720a2..be38a02 100644 --- a/src/main/resources/AGENTS.md +++ b/src/main/resources/AGENTS.md @@ -1,3 +1,5 @@ +> **Do not modify this file.** Sopa overwrites it whenever it migrates the project to a newer version, so any change here is lost. + # Sopa SOAP test project Plain YAML files. App watches folder, sees edits live. @@ -32,6 +34,7 @@ endpoints: version: 1.1 # or 1.2 headers: {} name: Test Project +version: "2" # of the project files, the app migrates older ones: never change it ``` testcase.yml diff --git a/src/test/java/com/molnarandrei/sopa/migrations/MigrationTest.java b/src/test/java/com/molnarandrei/sopa/migrations/MigrationTest.java new file mode 100644 index 0000000..831b04c --- /dev/null +++ b/src/test/java/com/molnarandrei/sopa/migrations/MigrationTest.java @@ -0,0 +1,68 @@ +package com.molnarandrei.sopa.migrations; + +import com.molnarandrei.sopa.data.yaml.Project; +import com.molnarandrei.sopa.data.yaml.TestCase; +import com.molnarandrei.sopa.project.ProjectRepository; +import com.molnarandrei.sopa.properties.PropertiesRepository; +import com.molnarandrei.sopa.testcase.TestCaseRepository; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Map; + +import static org.junit.jupiter.api.Assertions.*; + +public class MigrationTest { + @TempDir + Path folder; + + @Test + public void versionIsTheWholeNumberBeforeAnyDot() { + assertEquals(1, Migration.versionOf(Map.of("version", "1.0.2"))); + assertEquals(1, Migration.versionOf(Map.of("version", 1))); + assertEquals(2, Migration.versionOf(Map.of("version", "2"))); + assertEquals(1, Migration.versionOf(Map.of())); + } + + @Test + public void propertiesMoveToTheirFilesAndTheProjectIsCurrent() throws Exception { + var projectPath = folder.resolve("project.yml"); + Files.writeString(projectPath, Project.HEADER + """ + + version: 1.0.2 + endpoints: {} + headers: {} + name: P + properties: + host: localhost + """); + var testCasePath = folder.resolve("testcase-1").resolve("testcase.yml"); + Files.createDirectories(testCasePath.getParent()); + Files.writeString(testCasePath, TestCase.HEADER + """ + + name: C + properties: + alumnusAge: 20 + alumnusName: Zoe + """); + // Written by a newer version, so it wins over the yml. + Files.writeString(PropertiesRepository.of(testCasePath), "alumnusName=Ana\n"); + Files.writeString(folder.resolve("AGENTS.md"), "edited by the user"); + + Migration.migrate(projectPath); + + assertEquals(String.valueOf(Migration.VERSION), ProjectRepository.get(projectPath).getVersion()); + assertEquals(Map.of("host", "localhost"), PropertiesRepository.get(PropertiesRepository.of(projectPath))); + assertEquals(Map.of("alumnusAge", "20", "alumnusName", "Ana"), PropertiesRepository.get(PropertiesRepository.of(testCasePath))); + assertFalse(Files.readString(testCasePath).contains("properties")); + assertEquals("C", TestCaseRepository.get(testCasePath).getName()); + assertTrue(Files.readString(folder.resolve("AGENTS.md")).startsWith("> **Do not modify this file.**")); + + // A current project is left as it is. + Files.writeString(folder.resolve("AGENTS.md"), "untouched"); + Migration.migrate(projectPath); + assertEquals("untouched", Files.readString(folder.resolve("AGENTS.md"))); + } +} diff --git a/src/test/java/com/molnarandrei/sopa/project/ProjectFolderTest.java b/src/test/java/com/molnarandrei/sopa/project/ProjectFolderTest.java index b886779..39bcb25 100644 --- a/src/test/java/com/molnarandrei/sopa/project/ProjectFolderTest.java +++ b/src/test/java/com/molnarandrei/sopa/project/ProjectFolderTest.java @@ -45,7 +45,7 @@ public class ProjectFolderTest { @Test public void agentsFileIsWrittenIntoTheProject() throws Exception { ProjectRepository.writeAgentsFile(folder); - assertTrue(Files.readString(folder.resolve("AGENTS.md")).startsWith("# Sopa")); + assertTrue(Files.readString(folder.resolve("AGENTS.md")).startsWith("> **Do not modify this file.**")); } @Test