diff options
author | 2023-08-08 19:07:27 +0200 | |
---|---|---|
committer | 2023-08-08 19:07:27 +0200 | |
commit | 52c8e89faa49988bc47a15256ed818f3b78bb56b (patch) | |
tree | 3b97fffd54227c3cb155312467c0ffa625788710 /subprojects/store/src/test | |
parent | Non-lazy NeighbourhoodCalculator for more accurate StateCoderBuilderImpl. (diff) | |
download | refinery-52c8e89faa49988bc47a15256ed818f3b78bb56b.tar.gz refinery-52c8e89faa49988bc47a15256ed818f3b78bb56b.tar.zst refinery-52c8e89faa49988bc47a15256ed818f3b78bb56b.zip |
Test cases for Equivalence accuracy measurements
Diffstat (limited to 'subprojects/store/src/test')
-rw-r--r-- | subprojects/store/src/test/java/tools/refinery/store/statecoding/ExperimentalSetupTest.java | 189 |
1 files changed, 140 insertions, 49 deletions
diff --git a/subprojects/store/src/test/java/tools/refinery/store/statecoding/ExperimentalSetupTest.java b/subprojects/store/src/test/java/tools/refinery/store/statecoding/ExperimentalSetupTest.java index 87b1623c..25b5dca1 100644 --- a/subprojects/store/src/test/java/tools/refinery/store/statecoding/ExperimentalSetupTest.java +++ b/subprojects/store/src/test/java/tools/refinery/store/statecoding/ExperimentalSetupTest.java | |||
@@ -5,16 +5,52 @@ | |||
5 | */ | 5 | */ |
6 | package tools.refinery.store.statecoding; | 6 | package tools.refinery.store.statecoding; |
7 | 7 | ||
8 | import org.eclipse.collections.impl.map.mutable.primitive.IntObjectHashMap; | ||
9 | import org.junit.jupiter.api.Tag; | ||
8 | import org.junit.jupiter.api.Test; | 10 | import org.junit.jupiter.api.Test; |
11 | import org.junit.jupiter.params.ParameterizedTest; | ||
12 | import org.junit.jupiter.params.provider.ValueSource; | ||
9 | import tools.refinery.store.map.Version; | 13 | import tools.refinery.store.map.Version; |
14 | import tools.refinery.store.model.Model; | ||
10 | import tools.refinery.store.model.ModelStore; | 15 | import tools.refinery.store.model.ModelStore; |
11 | import tools.refinery.store.representation.Symbol; | 16 | import tools.refinery.store.representation.Symbol; |
12 | import tools.refinery.store.tuple.Tuple; | 17 | import tools.refinery.store.tuple.Tuple; |
13 | 18 | ||
14 | import java.util.*; | 19 | import java.util.ArrayList; |
20 | import java.util.HashSet; | ||
21 | import java.util.List; | ||
22 | import java.util.Set; | ||
23 | |||
24 | import static org.junit.jupiter.api.Assertions.assertEquals; | ||
25 | import static org.junit.jupiter.api.Assertions.assertTrue; | ||
26 | |||
15 | 27 | ||
16 | class ExperimentalSetupTest { | 28 | class ExperimentalSetupTest { |
17 | public static void generate(int size) { | 29 | static class ExperimentalSetupResult { |
30 | int versions = 0; | ||
31 | int different = 0; | ||
32 | int isomorphic = 0; | ||
33 | int unknown = 0; | ||
34 | |||
35 | double failureRatio() { | ||
36 | return (different + 0.0) / versions; | ||
37 | } | ||
38 | |||
39 | @Override | ||
40 | public String toString() { | ||
41 | return "ExperimentalSetupResult{" + | ||
42 | "versions=" + versions + | ||
43 | ", different=" + different + | ||
44 | ", isomorphic=" + isomorphic + | ||
45 | ", unknown=" + unknown + | ||
46 | ", ratio= " + failureRatio() + | ||
47 | '}'; | ||
48 | } | ||
49 | } | ||
50 | |||
51 | static int MAX = 100000; | ||
52 | |||
53 | public static ExperimentalSetupResult generate(int size, boolean permuteTypes) { | ||
18 | Symbol<Boolean> person = new Symbol<>("Person", 1, Boolean.class, false); | 54 | Symbol<Boolean> person = new Symbol<>("Person", 1, Boolean.class, false); |
19 | Symbol<Boolean> friend = new Symbol<>("friend", 2, Boolean.class, false); | 55 | Symbol<Boolean> friend = new Symbol<>("friend", 2, Boolean.class, false); |
20 | 56 | ||
@@ -25,13 +61,13 @@ class ExperimentalSetupTest { | |||
25 | .build(); | 61 | .build(); |
26 | 62 | ||
27 | Set<Version> versions = new HashSet<>(); | 63 | Set<Version> versions = new HashSet<>(); |
28 | Map<Integer, List<Version>> codes = new HashMap<>(); | 64 | IntObjectHashMap<List<Version>> codes = new IntObjectHashMap<>(); |
29 | 65 | ||
30 | var empty = store.createEmptyModel(); | 66 | var empty = store.createEmptyModel(); |
31 | var pI = empty.getInterpretation(person); | 67 | if (!permuteTypes) { |
32 | 68 | for (int i = 0; i < size; i++) { | |
33 | for (int i = 0; i < size; i++) { | 69 | empty.getInterpretation(person).put(Tuple.of(i), true); |
34 | pI.put(Tuple.of(i), true); | 70 | } |
35 | } | 71 | } |
36 | 72 | ||
37 | var emptyVersion = empty.commit(); | 73 | var emptyVersion = empty.commit(); |
@@ -42,12 +78,27 @@ class ExperimentalSetupTest { | |||
42 | codes.put(emptyCode, emptyList); | 78 | codes.put(emptyCode, emptyList); |
43 | 79 | ||
44 | var storeAdapter = store.getAdapter(StateCoderStoreAdapter.class); | 80 | var storeAdapter = store.getAdapter(StateCoderStoreAdapter.class); |
81 | var result = new ExperimentalSetupResult(); | ||
45 | 82 | ||
46 | int dif = 0; | 83 | int steps = 0; |
47 | int iso = 0; | ||
48 | int unk = 0; | ||
49 | 84 | ||
50 | //int step = 0 | 85 | if (permuteTypes) { |
86 | for (int i = 0; i < size; i++) { | ||
87 | var previousVersions = new HashSet<>(versions); | ||
88 | for (var version : previousVersions) { | ||
89 | var model = store.createModelForState(version); | ||
90 | model.getInterpretation(person).put(Tuple.of(i), true); | ||
91 | |||
92 | saveAsNewVersion(versions, codes, storeAdapter, result, model); | ||
93 | |||
94 | logProgress(steps++); | ||
95 | if (steps > MAX) { | ||
96 | result.versions = versions.size(); | ||
97 | return result; | ||
98 | } | ||
99 | } | ||
100 | } | ||
101 | } | ||
51 | 102 | ||
52 | for (int i = 0; i < size; i++) { | 103 | for (int i = 0; i < size; i++) { |
53 | for (int j = 0; j < size; j++) { | 104 | for (int j = 0; j < size; j++) { |
@@ -57,51 +108,91 @@ class ExperimentalSetupTest { | |||
57 | var model = store.createModelForState(version); | 108 | var model = store.createModelForState(version); |
58 | model.getInterpretation(friend).put(Tuple.of(i, j), true); | 109 | model.getInterpretation(friend).put(Tuple.of(i, j), true); |
59 | 110 | ||
60 | Version version1 = model.commit(); | 111 | saveAsNewVersion(versions, codes, storeAdapter, result, model); |
61 | var stateCode = model.getAdapter(StateCoderAdapter.class).calculateStateCode(); | 112 | |
62 | int code = stateCode.modelCode(); | 113 | logProgress(steps++); |
63 | //System.out.println(step+++" ->" +code); | 114 | if (steps > MAX) { |
64 | if (codes.containsKey(code)) { | 115 | result.versions = versions.size(); |
65 | Version similar = codes.get(code).get(0); | 116 | return result; |
66 | |||
67 | var outcome = storeAdapter.checkEquivalence(version1, similar); | ||
68 | if (outcome == StateEquivalenceChecker.EquivalenceResult.DIFFERENT) { | ||
69 | System.out.println(); | ||
70 | var c = model.getInterpretation(friend).getAll(); | ||
71 | while (c.move()) { | ||
72 | System.out.println(c.getKey().toString()); | ||
73 | } | ||
74 | System.out.println("vs"); | ||
75 | var c2 = store.createModelForState(similar).getInterpretation(friend).getAll(); | ||
76 | while (c2.move()) { | ||
77 | System.out.println(c2.getKey().toString()); | ||
78 | } | ||
79 | |||
80 | dif++; | ||
81 | } else if (outcome == StateEquivalenceChecker.EquivalenceResult.UNKNOWN) { | ||
82 | unk++; | ||
83 | } else { | ||
84 | iso++; | ||
85 | } | ||
86 | } else { | ||
87 | versions.add(version1); | ||
88 | |||
89 | List<Version> newList = new ArrayList<>(); | ||
90 | newList.add(version1); | ||
91 | codes.put(code, newList); | ||
92 | } | 117 | } |
93 | } | 118 | } |
94 | } | 119 | } |
95 | } | 120 | } |
96 | 121 | ||
97 | System.out.printf("v=%d i=%d d=%d u=%d\n", versions.size(), iso, dif, unk); | 122 | result.versions = versions.size(); |
123 | return result; | ||
98 | } | 124 | } |
99 | 125 | ||
100 | @Test | 126 | private static void saveAsNewVersion(Set<Version> versions, IntObjectHashMap<List<Version>> codes, |
101 | void runTests() { | 127 | StateCoderStoreAdapter storeAdapter, ExperimentalSetupResult result, Model model) { |
102 | for (int i = 0; i < 5; i++) { | 128 | Version version1 = model.commit(); |
103 | System.out.println("size = " + i); | 129 | |
104 | generate(i); | 130 | var stateCode = model.getAdapter(StateCoderAdapter.class).calculateStateCode(); |
131 | int code = stateCode.modelCode(); | ||
132 | if (codes.containsKey(code)) { | ||
133 | Version similar = codes.get(code).get(0); | ||
134 | |||
135 | var outcome = storeAdapter.checkEquivalence(version1, similar); | ||
136 | if (outcome == StateEquivalenceChecker.EquivalenceResult.DIFFERENT) { | ||
137 | result.different++; | ||
138 | } else if (outcome == StateEquivalenceChecker.EquivalenceResult.UNKNOWN) { | ||
139 | result.unknown++; | ||
140 | } else { | ||
141 | result.isomorphic++; | ||
142 | } | ||
143 | } else { | ||
144 | versions.add(version1); | ||
145 | |||
146 | List<Version> newList = new ArrayList<>(); | ||
147 | newList.add(version1); | ||
148 | codes.put(code, newList); | ||
149 | } | ||
150 | } | ||
151 | |||
152 | private static void logProgress(int steps) { | ||
153 | if (steps % 10000 == 0) { | ||
154 | System.out.println("Steps: " + steps + " / " + MAX); | ||
105 | } | 155 | } |
106 | } | 156 | } |
157 | |||
158 | static final double limit = 0.01; | ||
159 | |||
160 | @Test | ||
161 | void test0() { | ||
162 | assertEquals(1, generate(0, true).versions); | ||
163 | } | ||
164 | |||
165 | @ParameterizedTest | ||
166 | @ValueSource(ints = {1, 2, 3, 4}) | ||
167 | void testForSmallUntypedModels(int size) { | ||
168 | var res = generate(size, false); | ||
169 | System.out.println(res); | ||
170 | assertTrue(res.failureRatio() < limit); | ||
171 | } | ||
172 | |||
173 | @ParameterizedTest | ||
174 | @ValueSource(ints = {1, 2, 3}) | ||
175 | void testForSmallTypedModels(int size) { | ||
176 | var res = generate(size, true); | ||
177 | System.out.println(res); | ||
178 | assertTrue(res.failureRatio() < limit); | ||
179 | } | ||
180 | |||
181 | @Test | ||
182 | @Tag("fuzz") | ||
183 | @Tag("slow") | ||
184 | void testForLargeTypedModels() { | ||
185 | var res = generate(10, true); | ||
186 | System.out.println(res); | ||
187 | assertTrue(res.failureRatio() < limit); | ||
188 | } | ||
189 | |||
190 | @Test | ||
191 | @Tag("fuzz") | ||
192 | @Tag("slow") | ||
193 | void testForLargeUntypedModels() { | ||
194 | var res = generate(10, false); | ||
195 | System.out.println(res); | ||
196 | assertTrue(res.failureRatio() < limit); | ||
197 | } | ||
107 | } | 198 | } |