Migration system + separate properties migration.

This commit is contained in:
andrei committed 2026-11-19 09:58:48 +01:00
1 parent 8e64749111
commit 8591df9dbd
11 files changed
+227 -22

No files matched your search

@@ -1,6 +1,7 @@
package com.molnarandrei.sopa.data.yaml; package com.molnarandrei.sopa.data.yaml;
import com.molnarandrei.sopa.migrations.Migration;
import com.molnarandrei.sopa.data.ProjectMonitors; import com.molnarandrei.sopa.data.ProjectMonitors;
import com.molnarandrei.sopa.data.RepositoryException; import com.molnarandrei.sopa.data.RepositoryException;
import com.molnarandrei.sopa.events.Events; import com.molnarandrei.sopa.events.Events;
@@ -130,6 +131,7 @@ public class Config {
// So: iterate a copy, and only start listening once the saved projects are loaded. // So: iterate a copy, and only start listening once the saved projects are loaded.
for (var project : List.copyOf(loadedProjects)) { for (var project : List.copyOf(loadedProjects)) {
try { try {
Migration.migrate(project);
ProjectMonitors.start(project.getParent()); ProjectMonitors.start(project.getParent());
} catch (RepositoryException logAndIgnore) { } catch (RepositoryException logAndIgnore) {
log.log(Level.WARNING, "Cannot load previously loaded project at %s.".formatted(project), logAndIgnore); log.log(Level.WARNING, "Cannot load previously loaded project at %s.".formatted(project), logAndIgnore);
@@ -1,5 +1,6 @@
package com.molnarandrei.sopa.data.yaml; package com.molnarandrei.sopa.data.yaml;
import com.molnarandrei.sopa.migrations.Migration;
import com.molnarandrei.sopa.soap.SoapVersion; import com.molnarandrei.sopa.soap.SoapVersion;
import java.util.HashMap; import java.util.HashMap;
@@ -9,7 +10,8 @@ import java.util.Map;
public class Project { public class Project {
public static final String HEADER = "# com.molnarandrei.sopa.Project"; public static final String HEADER = "# com.molnarandrei.sopa.Project";
private String version = "1.0.2"; /** Of the files of the project: {@link Migration} brings older projects to the current one. */
private String version = String.valueOf(Migration.VERSION);
private String name = ""; private String name = "";
/** By url, in the order they were added. */ /** By url, in the order they were added. */
private Map<String, Endpoint> endpoints = new LinkedHashMap<>(); private Map<String, Endpoint> endpoints = new LinkedHashMap<>();
@@ -3,6 +3,9 @@ package com.molnarandrei.sopa.data.yaml;
import tools.jackson.core.JacksonException; import tools.jackson.core.JacksonException;
import tools.jackson.dataformat.yaml.YAMLMapper; import tools.jackson.dataformat.yaml.YAMLMapper;
import java.util.LinkedHashMap;
import java.util.Map;
public class Yaml { public class Yaml {
private static final YAMLMapper mapper = HumanComfyYamlMapper.get(); private static final YAMLMapper mapper = HumanComfyYamlMapper.get();
@@ -62,6 +65,17 @@ public class Yaml {
return Config.HEADER + '\n' + mapper.writeValueAsString(config); return Config.HEADER + '\n' + mapper.writeValueAsString(config);
} }
/** Any yml as plain maps, lists and scalars, as migrations read the files of older versions. */
@SuppressWarnings("unchecked")
public static Map<String, Object> toMap(String value) throws JacksonException {
var map = mapper.readValue(value, LinkedHashMap.class);
return map == null ? new LinkedHashMap<>() : map;
}
public static String toYaml(String header, Map<String, Object> map) {
return header + '\n' + mapper.writeValueAsString(map);
}
/** Whether both would be written the same, whatever the order of their maps. */ /** Whether both would be written the same, whatever the order of their maps. */
public static boolean sameContent(Object a, Object b) { public static boolean sameContent(Object a, Object b) {
return mapper.valueToTree(a).equals(mapper.valueToTree(b)); return mapper.valueToTree(a).equals(mapper.valueToTree(b));
@@ -0,0 +1,36 @@
package com.molnarandrei.sopa.migrations;
import com.molnarandrei.sopa.data.FileNames;
import com.molnarandrei.sopa.data.yaml.Yaml;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.List;
import java.util.Map;
/** File helpers for migrations, which read the files of older versions as plain maps rather than as the app objects. */
public class MigrateUtils {
/** The testcase folders in {@code projectFolder}, in no particular order. */
public static List<Path> testCaseFolders(Path projectFolder) throws IOException {
try (var children = Files.list(projectFolder)) {
return children.filter(folder -> Files.isDirectory(folder) && FileNames.hasTestCaseFolderName(folder)).toList();
}
}
/** The step folders in {@code testCaseFolder}, in no particular order. */
public static List<Path> stepFolders(Path testCaseFolder) throws IOException {
try (var children = Files.list(testCaseFolder)) {
return children.filter(FileNames::isStepFolder).toList();
}
}
public static Map<String, Object> readYaml(Path ymlPath) throws IOException {
return Yaml.toMap(Files.readString(ymlPath));
}
public static void writeYaml(Path ymlPath, String header, Map<String, Object> yaml) throws IOException {
Files.writeString(ymlPath, Yaml.toYaml(header, yaml));
}
}
@@ -0,0 +1,84 @@
package com.molnarandrei.sopa.migrations;
import com.molnarandrei.sopa.data.FileSystemCache;
import com.molnarandrei.sopa.data.RepositoryException;
import com.molnarandrei.sopa.data.yaml.Project;
import com.molnarandrei.sopa.data.yaml.TestCase;
import com.molnarandrei.sopa.project.ProjectRepository;
import com.molnarandrei.sopa.properties.PropertiesRepository;
import tools.jackson.core.JacksonException;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.Map;
import java.util.logging.Logger;
/**
* Brings the files of a project up to the {@link #VERSION} of this app, one version at a time, before anything else
* reads them. Migration {@code n} takes a project from version {@code n} to {@code n + 1}.
*/
public class Migration {
private static final Logger log = Logger.getLogger(Migration.class.getName());
/** The version of the projects of this app. */
public static final int VERSION = 2;
private Migration() {}
/** Migrates the project of {@code projectPath}, its project.yml, when its version is older. Overwrites its AGENTS.md. */
public static void migrate(Path projectPath) throws RepositoryException {
try {
var version = versionOf(MigrateUtils.readYaml(projectPath));
if (version >= VERSION) return;
var projectFolder = projectPath.getParent();
log.info("Migrating project %s from version %d to %d.".formatted(projectPath, version, VERSION));
for (; version < VERSION; version++)
switch (version) {
case 1 -> migration_1(projectFolder);
default -> throw new IllegalStateException("No migration from version %d".formatted(version));
}
var project = MigrateUtils.readYaml(projectPath);
project.put("version", String.valueOf(VERSION));
MigrateUtils.writeYaml(projectPath, Project.HEADER, project);
ProjectRepository.writeAgentsFile(projectFolder);
FileSystemCache.remove(projectFolder);
} catch (IOException | JacksonException cannotMigrate) {
throw new RepositoryException(projectPath, "Cannot migrate project", cannotMigrate);
}
}
/** The whole number before any dot: "1.0.2" is 1. A project without version is 1. */
static int versionOf(Map<String, Object> project) {
var version = project.get("version");
if (version == null) return 1;
return Integer.parseInt(String.valueOf(version).split("\\.")[0].strip());
}
/** Properties move from the yml of the project and its testcases to their .properties files. */
static void migration_1(Path projectFolder) throws IOException, RepositoryException {
var ymlPaths = new ArrayList<Path>();
ymlPaths.add(projectFolder.resolve("project.yml"));
for (var testCaseFolder : MigrateUtils.testCaseFolders(projectFolder))
ymlPaths.add(testCaseFolder.resolve("testcase.yml"));
for (var ymlPath : ymlPaths) {
if (! Files.exists(ymlPath)) continue;
var yaml = MigrateUtils.readYaml(ymlPath);
if (! (yaml.remove("properties") instanceof Map<?, ?> moved)) continue;
log.info("Moving testcase properties to .properties file %s".formatted(ymlPath));
var properties = new HashMap<String, String>();
moved.forEach((key, value) -> properties.put(String.valueOf(key), value == null ? "" : String.valueOf(value)));
// A properties file already there was written by a newer version: its values win.
var propertiesPath = PropertiesRepository.of(ymlPath);
properties.putAll(PropertiesRepository.get(propertiesPath));
PropertiesRepository.store(propertiesPath, properties);
MigrateUtils.writeYaml(ymlPath, ymlPath.endsWith("project.yml") ? Project.HEADER : TestCase.HEADER, yaml);
}
}
}
@@ -16,10 +16,9 @@ import tools.jackson.core.JacksonException;
import java.io.IOException; import java.io.IOException;
import java.nio.file.Files; import java.nio.file.Files;
import java.nio.file.Path; import java.nio.file.Path;
import java.nio.file.StandardCopyOption;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Comparator;
import java.util.List; import java.util.List;
import java.util.logging.Logger;
public class ProjectRepository { public class ProjectRepository {
@@ -115,12 +114,13 @@ public class ProjectRepository {
} }
} }
/** Writes the AGENTS.md of this version of the app, over any there is. */
public static void writeAgentsFile(Path projectDir) throws RepositoryException { public static void writeAgentsFile(Path projectDir) throws RepositoryException {
var agentsFile = projectDir.resolve("AGENTS.md"); var agentsFile = projectDir.resolve("AGENTS.md");
try (var template = ProjectRepository.class.getResourceAsStream("/AGENTS.md")) { try (var template = ProjectRepository.class.getResourceAsStream("/AGENTS.md")) {
if (template == null) if (template == null)
throw new IOException("AGENTS.md is not packaged"); throw new IOException("AGENTS.md is not packaged");
Files.copy(template, agentsFile); Files.copy(template, agentsFile, StandardCopyOption.REPLACE_EXISTING);
} catch (IOException io) { } catch (IOException io) {
throw new RepositoryException(agentsFile, "Could not create agents file", io); throw new RepositoryException(agentsFile, "Could not create agents file", io);
} }
@@ -2,25 +2,20 @@ package com.molnarandrei.sopa.project;
import java.nio.file.Files; import java.nio.file.Files;
import com.molnarandrei.sopa.data.yaml.Kind; import com.molnarandrei.sopa.data.yaml.Kind;
import com.molnarandrei.sopa.migrations.Migration;
import com.molnarandrei.sopa.undo.Undo; import com.molnarandrei.sopa.undo.Undo;
import com.molnarandrei.sopa.Utils; import com.molnarandrei.sopa.Utils;
import com.molnarandrei.sopa.data.ProjectMonitors; import com.molnarandrei.sopa.data.ProjectMonitors;
import com.molnarandrei.sopa.data.yaml.Project; import com.molnarandrei.sopa.data.yaml.Project;
import com.molnarandrei.sopa.data.yaml.Step;
import com.molnarandrei.sopa.data.yaml.TestCase;
import com.molnarandrei.sopa.project.ProjectRepository;
import com.molnarandrei.sopa.data.RepositoryException; import com.molnarandrei.sopa.data.RepositoryException;
import com.molnarandrei.sopa.events.Events; import com.molnarandrei.sopa.events.Events;
import com.molnarandrei.sopa.events.ProjectEvent; import com.molnarandrei.sopa.events.ProjectEvent;
import com.molnarandrei.sopa.step.StepRepository;
import com.molnarandrei.sopa.testcase.TestCaseRepository;
import java.nio.file.Path; import java.nio.file.Path;
import java.util.logging.Level; import java.util.logging.Level;
import java.util.logging.Logger; import java.util.logging.Logger;
import static com.molnarandrei.sopa.Utils.renameIndexed; import static com.molnarandrei.sopa.Utils.renameIndexed;
import static com.molnarandrei.sopa.data.yaml.Kind.project;
/** /**
* All operations over projects at the application level belong here. * All operations over projects at the application level belong here.
@@ -181,8 +176,10 @@ public class ProjectService {
return true; return true;
} }
/** Migrates the project to this version of the app, before anything reads it, and opens it. */
public static void open(Path projectPath) { public static void open(Path projectPath) {
try { try {
Migration.migrate(projectPath);
ProjectRepository.get(projectPath); ProjectRepository.get(projectPath);
ProjectMonitors.start(projectPath.getParent()); ProjectMonitors.start(projectPath.getParent());
} catch (RepositoryException repo) { } catch (RepositoryException repo) {
@@ -1,16 +1,10 @@
package com.molnarandrei.sopa.properties; package com.molnarandrei.sopa.properties;
import com.molnarandrei.sopa.data.ProjectMonitors; import com.molnarandrei.sopa.data.*;
import com.molnarandrei.sopa.data.RepositoryException;
import java.io.IOException; import java.io.*;
import java.io.StringReader; import java.nio.file.*;
import java.io.StringWriter; import java.util.*;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.HashMap;
import java.util.Map;
import java.util.Properties;
/** /**
* The properties of a project or testcase, in a java .properties file next to its yml: * The properties of a project or testcase, in a java .properties file next to its yml:
@@ -46,13 +40,18 @@ public class PropertiesRepository {
/** Writes {@code properties}, unless the file already holds them. */ /** Writes {@code properties}, unless the file already holds them. */
public static void update(Path propertiesPath, Map<String, String> properties) throws RepositoryException { public static void update(Path propertiesPath, Map<String, String> properties) throws RepositoryException {
if (get(propertiesPath).equals(properties)) return; if (get(propertiesPath).equals(properties)) return;
store(propertiesPath, properties);
ProjectMonitors.trigger(projectDirectoryOf(propertiesPath));
}
/** Writes {@code properties} without notifying the monitor: for projects not open yet, as when migrating. */
public static void store(Path propertiesPath, Map<String, String> properties) throws RepositoryException {
try { try {
var stored = new Properties(); var stored = new Properties();
stored.putAll(properties); stored.putAll(properties);
var text = new StringWriter(); var text = new StringWriter();
stored.store(text, null); stored.store(text, null);
Files.writeString(propertiesPath, text.toString()); Files.writeString(propertiesPath, text.toString());
ProjectMonitors.trigger(projectDirectoryOf(propertiesPath));
} catch (IOException io) { } catch (IOException io) {
throw new RepositoryException(propertiesPath, "Cannot write properties", io); throw new RepositoryException(propertiesPath, "Cannot write properties", io);
} }
+3
View File
@@ -1,3 +1,5 @@
> **Do not modify this file.** Sopa overwrites it whenever it migrates the project to a newer version, so any change here is lost.
# Sopa SOAP test project # Sopa SOAP test project
Plain YAML files. App watches folder, sees edits live. Plain YAML files. App watches folder, sees edits live.
@@ -32,6 +34,7 @@ endpoints:
version: 1.1 # or 1.2 version: 1.1 # or 1.2
headers: {} headers: {}
name: Test Project name: Test Project
version: "2" # of the project files, the app migrates older ones: never change it
``` ```
testcase.yml testcase.yml
@@ -0,0 +1,68 @@
package com.molnarandrei.sopa.migrations;
import com.molnarandrei.sopa.data.yaml.Project;
import com.molnarandrei.sopa.data.yaml.TestCase;
import com.molnarandrei.sopa.project.ProjectRepository;
import com.molnarandrei.sopa.properties.PropertiesRepository;
import com.molnarandrei.sopa.testcase.TestCaseRepository;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.io.TempDir;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.Map;
import static org.junit.jupiter.api.Assertions.*;
public class MigrationTest {
@TempDir
Path folder;
@Test
public void versionIsTheWholeNumberBeforeAnyDot() {
assertEquals(1, Migration.versionOf(Map.of("version", "1.0.2")));
assertEquals(1, Migration.versionOf(Map.of("version", 1)));
assertEquals(2, Migration.versionOf(Map.of("version", "2")));
assertEquals(1, Migration.versionOf(Map.of()));
}
@Test
public void propertiesMoveToTheirFilesAndTheProjectIsCurrent() throws Exception {
var projectPath = folder.resolve("project.yml");
Files.writeString(projectPath, Project.HEADER + """
version: 1.0.2
endpoints: {}
headers: {}
name: P
properties:
host: localhost
""");
var testCasePath = folder.resolve("testcase-1").resolve("testcase.yml");
Files.createDirectories(testCasePath.getParent());
Files.writeString(testCasePath, TestCase.HEADER + """
name: C
properties:
alumnusAge: 20
alumnusName: Zoe
""");
// Written by a newer version, so it wins over the yml.
Files.writeString(PropertiesRepository.of(testCasePath), "alumnusName=Ana\n");
Files.writeString(folder.resolve("AGENTS.md"), "edited by the user");
Migration.migrate(projectPath);
assertEquals(String.valueOf(Migration.VERSION), ProjectRepository.get(projectPath).getVersion());
assertEquals(Map.of("host", "localhost"), PropertiesRepository.get(PropertiesRepository.of(projectPath)));
assertEquals(Map.of("alumnusAge", "20", "alumnusName", "Ana"), PropertiesRepository.get(PropertiesRepository.of(testCasePath)));
assertFalse(Files.readString(testCasePath).contains("properties"));
assertEquals("C", TestCaseRepository.get(testCasePath).getName());
assertTrue(Files.readString(folder.resolve("AGENTS.md")).startsWith("> **Do not modify this file.**"));
// A current project is left as it is.
Files.writeString(folder.resolve("AGENTS.md"), "untouched");
Migration.migrate(projectPath);
assertEquals("untouched", Files.readString(folder.resolve("AGENTS.md")));
}
}
@@ -45,7 +45,7 @@ public class ProjectFolderTest {
@Test @Test
public void agentsFileIsWrittenIntoTheProject() throws Exception { public void agentsFileIsWrittenIntoTheProject() throws Exception {
ProjectRepository.writeAgentsFile(folder); ProjectRepository.writeAgentsFile(folder);
assertTrue(Files.readString(folder.resolve("AGENTS.md")).startsWith("# Sopa")); assertTrue(Files.readString(folder.resolve("AGENTS.md")).startsWith("> **Do not modify this file.**"));
} }
@Test @Test