Migration system + separate properties migration.
This commit is contained in:
1 parent
8e64749111
commit
8591df9dbd
11 files changed
+227
-22
No files matched your search
@@ -1,6 +1,7 @@
|
||||
package com.molnarandrei.sopa.data.yaml;
|
||||
|
||||
|
||||
import com.molnarandrei.sopa.migrations.Migration;
|
||||
import com.molnarandrei.sopa.data.ProjectMonitors;
|
||||
import com.molnarandrei.sopa.data.RepositoryException;
|
||||
import com.molnarandrei.sopa.events.Events;
|
||||
@@ -130,6 +131,7 @@ public class Config {
|
||||
// So: iterate a copy, and only start listening once the saved projects are loaded.
|
||||
for (var project : List.copyOf(loadedProjects)) {
|
||||
try {
|
||||
Migration.migrate(project);
|
||||
ProjectMonitors.start(project.getParent());
|
||||
} catch (RepositoryException logAndIgnore) {
|
||||
log.log(Level.WARNING, "Cannot load previously loaded project at %s.".formatted(project), logAndIgnore);
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
package com.molnarandrei.sopa.data.yaml;
|
||||
|
||||
import com.molnarandrei.sopa.migrations.Migration;
|
||||
import com.molnarandrei.sopa.soap.SoapVersion;
|
||||
|
||||
import java.util.HashMap;
|
||||
@@ -9,7 +10,8 @@ import java.util.Map;
|
||||
|
||||
public class Project {
|
||||
public static final String HEADER = "# com.molnarandrei.sopa.Project";
|
||||
private String version = "1.0.2";
|
||||
/** Of the files of the project: {@link Migration} brings older projects to the current one. */
|
||||
private String version = String.valueOf(Migration.VERSION);
|
||||
private String name = "";
|
||||
/** By url, in the order they were added. */
|
||||
private Map<String, Endpoint> endpoints = new LinkedHashMap<>();
|
||||
|
||||
@@ -3,6 +3,9 @@ package com.molnarandrei.sopa.data.yaml;
|
||||
import tools.jackson.core.JacksonException;
|
||||
import tools.jackson.dataformat.yaml.YAMLMapper;
|
||||
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.Map;
|
||||
|
||||
public class Yaml {
|
||||
private static final YAMLMapper mapper = HumanComfyYamlMapper.get();
|
||||
|
||||
@@ -62,6 +65,17 @@ public class Yaml {
|
||||
return Config.HEADER + '\n' + mapper.writeValueAsString(config);
|
||||
}
|
||||
|
||||
/** Any yml as plain maps, lists and scalars, as migrations read the files of older versions. */
|
||||
@SuppressWarnings("unchecked")
|
||||
public static Map<String, Object> toMap(String value) throws JacksonException {
|
||||
var map = mapper.readValue(value, LinkedHashMap.class);
|
||||
return map == null ? new LinkedHashMap<>() : map;
|
||||
}
|
||||
|
||||
public static String toYaml(String header, Map<String, Object> map) {
|
||||
return header + '\n' + mapper.writeValueAsString(map);
|
||||
}
|
||||
|
||||
/** Whether both would be written the same, whatever the order of their maps. */
|
||||
public static boolean sameContent(Object a, Object b) {
|
||||
return mapper.valueToTree(a).equals(mapper.valueToTree(b));
|
||||
|
||||
@@ -0,0 +1,36 @@
|
||||
package com.molnarandrei.sopa.migrations;
|
||||
|
||||
import com.molnarandrei.sopa.data.FileNames;
|
||||
import com.molnarandrei.sopa.data.yaml.Yaml;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
/** File helpers for migrations, which read the files of older versions as plain maps rather than as the app objects. */
|
||||
public class MigrateUtils {
|
||||
|
||||
/** The testcase folders in {@code projectFolder}, in no particular order. */
|
||||
public static List<Path> testCaseFolders(Path projectFolder) throws IOException {
|
||||
try (var children = Files.list(projectFolder)) {
|
||||
return children.filter(folder -> Files.isDirectory(folder) && FileNames.hasTestCaseFolderName(folder)).toList();
|
||||
}
|
||||
}
|
||||
|
||||
/** The step folders in {@code testCaseFolder}, in no particular order. */
|
||||
public static List<Path> stepFolders(Path testCaseFolder) throws IOException {
|
||||
try (var children = Files.list(testCaseFolder)) {
|
||||
return children.filter(FileNames::isStepFolder).toList();
|
||||
}
|
||||
}
|
||||
|
||||
public static Map<String, Object> readYaml(Path ymlPath) throws IOException {
|
||||
return Yaml.toMap(Files.readString(ymlPath));
|
||||
}
|
||||
|
||||
public static void writeYaml(Path ymlPath, String header, Map<String, Object> yaml) throws IOException {
|
||||
Files.writeString(ymlPath, Yaml.toYaml(header, yaml));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,84 @@
|
||||
package com.molnarandrei.sopa.migrations;
|
||||
|
||||
import com.molnarandrei.sopa.data.FileSystemCache;
|
||||
import com.molnarandrei.sopa.data.RepositoryException;
|
||||
import com.molnarandrei.sopa.data.yaml.Project;
|
||||
import com.molnarandrei.sopa.data.yaml.TestCase;
|
||||
import com.molnarandrei.sopa.project.ProjectRepository;
|
||||
import com.molnarandrei.sopa.properties.PropertiesRepository;
|
||||
import tools.jackson.core.JacksonException;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashMap;
|
||||
import java.util.Map;
|
||||
import java.util.logging.Logger;
|
||||
|
||||
/**
|
||||
* Brings the files of a project up to the {@link #VERSION} of this app, one version at a time, before anything else
|
||||
* reads them. Migration {@code n} takes a project from version {@code n} to {@code n + 1}.
|
||||
*/
|
||||
public class Migration {
|
||||
private static final Logger log = Logger.getLogger(Migration.class.getName());
|
||||
|
||||
/** The version of the projects of this app. */
|
||||
public static final int VERSION = 2;
|
||||
|
||||
private Migration() {}
|
||||
|
||||
/** Migrates the project of {@code projectPath}, its project.yml, when its version is older. Overwrites its AGENTS.md. */
|
||||
public static void migrate(Path projectPath) throws RepositoryException {
|
||||
try {
|
||||
var version = versionOf(MigrateUtils.readYaml(projectPath));
|
||||
if (version >= VERSION) return;
|
||||
|
||||
var projectFolder = projectPath.getParent();
|
||||
log.info("Migrating project %s from version %d to %d.".formatted(projectPath, version, VERSION));
|
||||
for (; version < VERSION; version++)
|
||||
switch (version) {
|
||||
case 1 -> migration_1(projectFolder);
|
||||
default -> throw new IllegalStateException("No migration from version %d".formatted(version));
|
||||
}
|
||||
|
||||
var project = MigrateUtils.readYaml(projectPath);
|
||||
project.put("version", String.valueOf(VERSION));
|
||||
MigrateUtils.writeYaml(projectPath, Project.HEADER, project);
|
||||
ProjectRepository.writeAgentsFile(projectFolder);
|
||||
FileSystemCache.remove(projectFolder);
|
||||
} catch (IOException | JacksonException cannotMigrate) {
|
||||
throw new RepositoryException(projectPath, "Cannot migrate project", cannotMigrate);
|
||||
}
|
||||
}
|
||||
|
||||
/** The whole number before any dot: "1.0.2" is 1. A project without version is 1. */
|
||||
static int versionOf(Map<String, Object> project) {
|
||||
var version = project.get("version");
|
||||
if (version == null) return 1;
|
||||
return Integer.parseInt(String.valueOf(version).split("\\.")[0].strip());
|
||||
}
|
||||
|
||||
/** Properties move from the yml of the project and its testcases to their .properties files. */
|
||||
static void migration_1(Path projectFolder) throws IOException, RepositoryException {
|
||||
var ymlPaths = new ArrayList<Path>();
|
||||
ymlPaths.add(projectFolder.resolve("project.yml"));
|
||||
for (var testCaseFolder : MigrateUtils.testCaseFolders(projectFolder))
|
||||
ymlPaths.add(testCaseFolder.resolve("testcase.yml"));
|
||||
|
||||
for (var ymlPath : ymlPaths) {
|
||||
if (! Files.exists(ymlPath)) continue;
|
||||
var yaml = MigrateUtils.readYaml(ymlPath);
|
||||
if (! (yaml.remove("properties") instanceof Map<?, ?> moved)) continue;
|
||||
log.info("Moving testcase properties to .properties file %s".formatted(ymlPath));
|
||||
|
||||
var properties = new HashMap<String, String>();
|
||||
moved.forEach((key, value) -> properties.put(String.valueOf(key), value == null ? "" : String.valueOf(value)));
|
||||
// A properties file already there was written by a newer version: its values win.
|
||||
var propertiesPath = PropertiesRepository.of(ymlPath);
|
||||
properties.putAll(PropertiesRepository.get(propertiesPath));
|
||||
PropertiesRepository.store(propertiesPath, properties);
|
||||
MigrateUtils.writeYaml(ymlPath, ymlPath.endsWith("project.yml") ? Project.HEADER : TestCase.HEADER, yaml);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -16,10 +16,9 @@ import tools.jackson.core.JacksonException;
|
||||
import java.io.IOException;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.nio.file.StandardCopyOption;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Comparator;
|
||||
import java.util.List;
|
||||
import java.util.logging.Logger;
|
||||
|
||||
|
||||
public class ProjectRepository {
|
||||
@@ -115,12 +114,13 @@ public class ProjectRepository {
|
||||
}
|
||||
}
|
||||
|
||||
/** Writes the AGENTS.md of this version of the app, over any there is. */
|
||||
public static void writeAgentsFile(Path projectDir) throws RepositoryException {
|
||||
var agentsFile = projectDir.resolve("AGENTS.md");
|
||||
try (var template = ProjectRepository.class.getResourceAsStream("/AGENTS.md")) {
|
||||
if (template == null)
|
||||
throw new IOException("AGENTS.md is not packaged");
|
||||
Files.copy(template, agentsFile);
|
||||
Files.copy(template, agentsFile, StandardCopyOption.REPLACE_EXISTING);
|
||||
} catch (IOException io) {
|
||||
throw new RepositoryException(agentsFile, "Could not create agents file", io);
|
||||
}
|
||||
|
||||
@@ -2,25 +2,20 @@ package com.molnarandrei.sopa.project;
|
||||
|
||||
import java.nio.file.Files;
|
||||
import com.molnarandrei.sopa.data.yaml.Kind;
|
||||
import com.molnarandrei.sopa.migrations.Migration;
|
||||
import com.molnarandrei.sopa.undo.Undo;
|
||||
import com.molnarandrei.sopa.Utils;
|
||||
import com.molnarandrei.sopa.data.ProjectMonitors;
|
||||
import com.molnarandrei.sopa.data.yaml.Project;
|
||||
import com.molnarandrei.sopa.data.yaml.Step;
|
||||
import com.molnarandrei.sopa.data.yaml.TestCase;
|
||||
import com.molnarandrei.sopa.project.ProjectRepository;
|
||||
import com.molnarandrei.sopa.data.RepositoryException;
|
||||
import com.molnarandrei.sopa.events.Events;
|
||||
import com.molnarandrei.sopa.events.ProjectEvent;
|
||||
import com.molnarandrei.sopa.step.StepRepository;
|
||||
import com.molnarandrei.sopa.testcase.TestCaseRepository;
|
||||
|
||||
import java.nio.file.Path;
|
||||
import java.util.logging.Level;
|
||||
import java.util.logging.Logger;
|
||||
|
||||
import static com.molnarandrei.sopa.Utils.renameIndexed;
|
||||
import static com.molnarandrei.sopa.data.yaml.Kind.project;
|
||||
|
||||
/**
|
||||
* All operations over projects at the application level belong here.
|
||||
@@ -181,8 +176,10 @@ public class ProjectService {
|
||||
return true;
|
||||
}
|
||||
|
||||
/** Migrates the project to this version of the app, before anything reads it, and opens it. */
|
||||
public static void open(Path projectPath) {
|
||||
try {
|
||||
Migration.migrate(projectPath);
|
||||
ProjectRepository.get(projectPath);
|
||||
ProjectMonitors.start(projectPath.getParent());
|
||||
} catch (RepositoryException repo) {
|
||||
|
||||
@@ -1,16 +1,10 @@
|
||||
package com.molnarandrei.sopa.properties;
|
||||
|
||||
import com.molnarandrei.sopa.data.ProjectMonitors;
|
||||
import com.molnarandrei.sopa.data.RepositoryException;
|
||||
import com.molnarandrei.sopa.data.*;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.io.StringReader;
|
||||
import java.io.StringWriter;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.util.HashMap;
|
||||
import java.util.Map;
|
||||
import java.util.Properties;
|
||||
import java.io.*;
|
||||
import java.nio.file.*;
|
||||
import java.util.*;
|
||||
|
||||
/**
|
||||
* The properties of a project or testcase, in a java .properties file next to its yml:
|
||||
@@ -46,13 +40,18 @@ public class PropertiesRepository {
|
||||
/** Writes {@code properties}, unless the file already holds them. */
|
||||
public static void update(Path propertiesPath, Map<String, String> properties) throws RepositoryException {
|
||||
if (get(propertiesPath).equals(properties)) return;
|
||||
store(propertiesPath, properties);
|
||||
ProjectMonitors.trigger(projectDirectoryOf(propertiesPath));
|
||||
}
|
||||
|
||||
/** Writes {@code properties} without notifying the monitor: for projects not open yet, as when migrating. */
|
||||
public static void store(Path propertiesPath, Map<String, String> properties) throws RepositoryException {
|
||||
try {
|
||||
var stored = new Properties();
|
||||
stored.putAll(properties);
|
||||
var text = new StringWriter();
|
||||
stored.store(text, null);
|
||||
Files.writeString(propertiesPath, text.toString());
|
||||
ProjectMonitors.trigger(projectDirectoryOf(propertiesPath));
|
||||
} catch (IOException io) {
|
||||
throw new RepositoryException(propertiesPath, "Cannot write properties", io);
|
||||
}
|
||||
|
||||
@@ -1,3 +1,5 @@
|
||||
> **Do not modify this file.** Sopa overwrites it whenever it migrates the project to a newer version, so any change here is lost.
|
||||
|
||||
# Sopa SOAP test project
|
||||
|
||||
Plain YAML files. App watches folder, sees edits live.
|
||||
@@ -32,6 +34,7 @@ endpoints:
|
||||
version: 1.1 # or 1.2
|
||||
headers: {}
|
||||
name: Test Project
|
||||
version: "2" # of the project files, the app migrates older ones: never change it
|
||||
```
|
||||
|
||||
testcase.yml
|
||||
|
||||
@@ -0,0 +1,68 @@
|
||||
package com.molnarandrei.sopa.migrations;
|
||||
|
||||
import com.molnarandrei.sopa.data.yaml.Project;
|
||||
import com.molnarandrei.sopa.data.yaml.TestCase;
|
||||
import com.molnarandrei.sopa.project.ProjectRepository;
|
||||
import com.molnarandrei.sopa.properties.PropertiesRepository;
|
||||
import com.molnarandrei.sopa.testcase.TestCaseRepository;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.io.TempDir;
|
||||
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.util.Map;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.*;
|
||||
|
||||
public class MigrationTest {
|
||||
@TempDir
|
||||
Path folder;
|
||||
|
||||
@Test
|
||||
public void versionIsTheWholeNumberBeforeAnyDot() {
|
||||
assertEquals(1, Migration.versionOf(Map.of("version", "1.0.2")));
|
||||
assertEquals(1, Migration.versionOf(Map.of("version", 1)));
|
||||
assertEquals(2, Migration.versionOf(Map.of("version", "2")));
|
||||
assertEquals(1, Migration.versionOf(Map.of()));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void propertiesMoveToTheirFilesAndTheProjectIsCurrent() throws Exception {
|
||||
var projectPath = folder.resolve("project.yml");
|
||||
Files.writeString(projectPath, Project.HEADER + """
|
||||
|
||||
version: 1.0.2
|
||||
endpoints: {}
|
||||
headers: {}
|
||||
name: P
|
||||
properties:
|
||||
host: localhost
|
||||
""");
|
||||
var testCasePath = folder.resolve("testcase-1").resolve("testcase.yml");
|
||||
Files.createDirectories(testCasePath.getParent());
|
||||
Files.writeString(testCasePath, TestCase.HEADER + """
|
||||
|
||||
name: C
|
||||
properties:
|
||||
alumnusAge: 20
|
||||
alumnusName: Zoe
|
||||
""");
|
||||
// Written by a newer version, so it wins over the yml.
|
||||
Files.writeString(PropertiesRepository.of(testCasePath), "alumnusName=Ana\n");
|
||||
Files.writeString(folder.resolve("AGENTS.md"), "edited by the user");
|
||||
|
||||
Migration.migrate(projectPath);
|
||||
|
||||
assertEquals(String.valueOf(Migration.VERSION), ProjectRepository.get(projectPath).getVersion());
|
||||
assertEquals(Map.of("host", "localhost"), PropertiesRepository.get(PropertiesRepository.of(projectPath)));
|
||||
assertEquals(Map.of("alumnusAge", "20", "alumnusName", "Ana"), PropertiesRepository.get(PropertiesRepository.of(testCasePath)));
|
||||
assertFalse(Files.readString(testCasePath).contains("properties"));
|
||||
assertEquals("C", TestCaseRepository.get(testCasePath).getName());
|
||||
assertTrue(Files.readString(folder.resolve("AGENTS.md")).startsWith("> **Do not modify this file.**"));
|
||||
|
||||
// A current project is left as it is.
|
||||
Files.writeString(folder.resolve("AGENTS.md"), "untouched");
|
||||
Migration.migrate(projectPath);
|
||||
assertEquals("untouched", Files.readString(folder.resolve("AGENTS.md")));
|
||||
}
|
||||
}
|
||||
@@ -45,7 +45,7 @@ public class ProjectFolderTest {
|
||||
@Test
|
||||
public void agentsFileIsWrittenIntoTheProject() throws Exception {
|
||||
ProjectRepository.writeAgentsFile(folder);
|
||||
assertTrue(Files.readString(folder.resolve("AGENTS.md")).startsWith("# Sopa"));
|
||||
assertTrue(Files.readString(folder.resolve("AGENTS.md")).startsWith("> **Do not modify this file.**"));
|
||||
}
|
||||
|
||||
@Test
|
||||
|
||||
Reference in new issue
Block a user