From 52c8e89faa49988bc47a15256ed818f3b78bb56b Mon Sep 17 00:00:00 2001 From: OszkarSemerath Date: Tue, 8 Aug 2023 19:07:27 +0200 Subject: Test cases for Equivalence accuracy measurements --- .../store/statecoding/ExperimentalSetupTest.java | 189 +++++++++++++++------ 1 file changed, 140 insertions(+), 49 deletions(-) (limited to 'subprojects/store/src') diff --git a/subprojects/store/src/test/java/tools/refinery/store/statecoding/ExperimentalSetupTest.java b/subprojects/store/src/test/java/tools/refinery/store/statecoding/ExperimentalSetupTest.java index 87b1623c..25b5dca1 100644 --- a/subprojects/store/src/test/java/tools/refinery/store/statecoding/ExperimentalSetupTest.java +++ b/subprojects/store/src/test/java/tools/refinery/store/statecoding/ExperimentalSetupTest.java @@ -5,16 +5,52 @@ */ package tools.refinery.store.statecoding; +import org.eclipse.collections.impl.map.mutable.primitive.IntObjectHashMap; +import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; import tools.refinery.store.map.Version; +import tools.refinery.store.model.Model; import tools.refinery.store.model.ModelStore; import tools.refinery.store.representation.Symbol; import tools.refinery.store.tuple.Tuple; -import java.util.*; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + class ExperimentalSetupTest { - public static void generate(int size) { + static class ExperimentalSetupResult { + int versions = 0; + int different = 0; + int isomorphic = 0; + int unknown = 0; + + double failureRatio() { + return (different + 0.0) / versions; + } + + @Override + public String toString() { + return "ExperimentalSetupResult{" + + "versions=" + versions + + ", different=" + different + + ", isomorphic=" + isomorphic + + ", unknown=" + unknown + + ", ratio= " + failureRatio() + + '}'; + } + } + + static int MAX = 100000; + + public static ExperimentalSetupResult generate(int size, boolean permuteTypes) { Symbol person = new Symbol<>("Person", 1, Boolean.class, false); Symbol friend = new Symbol<>("friend", 2, Boolean.class, false); @@ -25,13 +61,13 @@ class ExperimentalSetupTest { .build(); Set versions = new HashSet<>(); - Map> codes = new HashMap<>(); + IntObjectHashMap> codes = new IntObjectHashMap<>(); var empty = store.createEmptyModel(); - var pI = empty.getInterpretation(person); - - for (int i = 0; i < size; i++) { - pI.put(Tuple.of(i), true); + if (!permuteTypes) { + for (int i = 0; i < size; i++) { + empty.getInterpretation(person).put(Tuple.of(i), true); + } } var emptyVersion = empty.commit(); @@ -42,12 +78,27 @@ class ExperimentalSetupTest { codes.put(emptyCode, emptyList); var storeAdapter = store.getAdapter(StateCoderStoreAdapter.class); + var result = new ExperimentalSetupResult(); - int dif = 0; - int iso = 0; - int unk = 0; + int steps = 0; - //int step = 0 + if (permuteTypes) { + for (int i = 0; i < size; i++) { + var previousVersions = new HashSet<>(versions); + for (var version : previousVersions) { + var model = store.createModelForState(version); + model.getInterpretation(person).put(Tuple.of(i), true); + + saveAsNewVersion(versions, codes, storeAdapter, result, model); + + logProgress(steps++); + if (steps > MAX) { + result.versions = versions.size(); + return result; + } + } + } + } for (int i = 0; i < size; i++) { for (int j = 0; j < size; j++) { @@ -57,51 +108,91 @@ class ExperimentalSetupTest { var model = store.createModelForState(version); model.getInterpretation(friend).put(Tuple.of(i, j), true); - Version version1 = model.commit(); - var stateCode = model.getAdapter(StateCoderAdapter.class).calculateStateCode(); - int code = stateCode.modelCode(); - //System.out.println(step+++" ->" +code); - if (codes.containsKey(code)) { - Version similar = codes.get(code).get(0); - - var outcome = storeAdapter.checkEquivalence(version1, similar); - if (outcome == StateEquivalenceChecker.EquivalenceResult.DIFFERENT) { - System.out.println(); - var c = model.getInterpretation(friend).getAll(); - while (c.move()) { - System.out.println(c.getKey().toString()); - } - System.out.println("vs"); - var c2 = store.createModelForState(similar).getInterpretation(friend).getAll(); - while (c2.move()) { - System.out.println(c2.getKey().toString()); - } - - dif++; - } else if (outcome == StateEquivalenceChecker.EquivalenceResult.UNKNOWN) { - unk++; - } else { - iso++; - } - } else { - versions.add(version1); - - List newList = new ArrayList<>(); - newList.add(version1); - codes.put(code, newList); + saveAsNewVersion(versions, codes, storeAdapter, result, model); + + logProgress(steps++); + if (steps > MAX) { + result.versions = versions.size(); + return result; } } } } - System.out.printf("v=%d i=%d d=%d u=%d\n", versions.size(), iso, dif, unk); + result.versions = versions.size(); + return result; } - @Test - void runTests() { - for (int i = 0; i < 5; i++) { - System.out.println("size = " + i); - generate(i); + private static void saveAsNewVersion(Set versions, IntObjectHashMap> codes, + StateCoderStoreAdapter storeAdapter, ExperimentalSetupResult result, Model model) { + Version version1 = model.commit(); + + var stateCode = model.getAdapter(StateCoderAdapter.class).calculateStateCode(); + int code = stateCode.modelCode(); + if (codes.containsKey(code)) { + Version similar = codes.get(code).get(0); + + var outcome = storeAdapter.checkEquivalence(version1, similar); + if (outcome == StateEquivalenceChecker.EquivalenceResult.DIFFERENT) { + result.different++; + } else if (outcome == StateEquivalenceChecker.EquivalenceResult.UNKNOWN) { + result.unknown++; + } else { + result.isomorphic++; + } + } else { + versions.add(version1); + + List newList = new ArrayList<>(); + newList.add(version1); + codes.put(code, newList); + } + } + + private static void logProgress(int steps) { + if (steps % 10000 == 0) { + System.out.println("Steps: " + steps + " / " + MAX); } } + + static final double limit = 0.01; + + @Test + void test0() { + assertEquals(1, generate(0, true).versions); + } + + @ParameterizedTest + @ValueSource(ints = {1, 2, 3, 4}) + void testForSmallUntypedModels(int size) { + var res = generate(size, false); + System.out.println(res); + assertTrue(res.failureRatio() < limit); + } + + @ParameterizedTest + @ValueSource(ints = {1, 2, 3}) + void testForSmallTypedModels(int size) { + var res = generate(size, true); + System.out.println(res); + assertTrue(res.failureRatio() < limit); + } + + @Test + @Tag("fuzz") + @Tag("slow") + void testForLargeTypedModels() { + var res = generate(10, true); + System.out.println(res); + assertTrue(res.failureRatio() < limit); + } + + @Test + @Tag("fuzz") + @Tag("slow") + void testForLargeUntypedModels() { + var res = generate(10, false); + System.out.println(res); + assertTrue(res.failureRatio() < limit); + } } -- cgit v1.2.3-54-g00ecf