diff options
Diffstat (limited to 'Tests/hu.bme.mit.inf.dslreasoner.run/src/hu/bme/mit/inf/dslreasoner/run/script')
2 files changed, 421 insertions, 0 deletions
diff --git a/Tests/hu.bme.mit.inf.dslreasoner.run/src/hu/bme/mit/inf/dslreasoner/run/script/MeasurementScript.xtend b/Tests/hu.bme.mit.inf.dslreasoner.run/src/hu/bme/mit/inf/dslreasoner/run/script/MeasurementScript.xtend new file mode 100644 index 00000000..5abff962 --- /dev/null +++ b/Tests/hu.bme.mit.inf.dslreasoner.run/src/hu/bme/mit/inf/dslreasoner/run/script/MeasurementScript.xtend | |||
@@ -0,0 +1,70 @@ | |||
1 | package hu.bme.mit.inf.dslreasoner.run.script | ||
2 | |||
3 | import java.util.List | ||
4 | import org.eclipse.xtend.lib.annotations.Accessors | ||
5 | |||
6 | @Accessors | ||
7 | class MeasurementScript { | ||
8 | String inputPath | ||
9 | String outputPath | ||
10 | int timeout | ||
11 | boolean saveModels | ||
12 | boolean saveTemporaryFiles | ||
13 | int warmupIterations | ||
14 | int iterations | ||
15 | Domain domain | ||
16 | Scope scope | ||
17 | List<Integer> sizes | ||
18 | Solver solver | ||
19 | ScopePropagator scopePropagator | ||
20 | ScopeConstraints propagatedConstraints | ||
21 | PolyhedronSolver polyhedronSolver | ||
22 | ScopeHeuristic scopeHeuristic | ||
23 | |||
24 | def toCsvHeader() { | ||
25 | '''«domain»,«scope»,«solver»,«scopePropagator ?: "NULL"»,«propagatedConstraints ?: "NULL"»,«polyhedronSolver ?: "NULL"»''' | ||
26 | } | ||
27 | } | ||
28 | |||
29 | enum Domain { | ||
30 | fs, | ||
31 | ecore, | ||
32 | Yakindu, | ||
33 | FAM, | ||
34 | satellite | ||
35 | } | ||
36 | |||
37 | enum Scope { | ||
38 | none, | ||
39 | quantiles | ||
40 | } | ||
41 | |||
42 | enum Solver { | ||
43 | ViatraSolver, | ||
44 | AlloySolver | ||
45 | } | ||
46 | |||
47 | enum ScopePropagator { | ||
48 | none, | ||
49 | basic, | ||
50 | polyhedral | ||
51 | } | ||
52 | |||
53 | enum ScopeConstraints { | ||
54 | none, | ||
55 | typeHierarchy, | ||
56 | relations, | ||
57 | hints | ||
58 | } | ||
59 | |||
60 | enum PolyhedronSolver { | ||
61 | Z3Integer, | ||
62 | Z3Real, | ||
63 | Cbc, | ||
64 | Clp | ||
65 | } | ||
66 | |||
67 | enum ScopeHeuristic { | ||
68 | basic, | ||
69 | polyhedral | ||
70 | } | ||
diff --git a/Tests/hu.bme.mit.inf.dslreasoner.run/src/hu/bme/mit/inf/dslreasoner/run/script/MeasurementScriptRunner.xtend b/Tests/hu.bme.mit.inf.dslreasoner.run/src/hu/bme/mit/inf/dslreasoner/run/script/MeasurementScriptRunner.xtend new file mode 100644 index 00000000..48e750cb --- /dev/null +++ b/Tests/hu.bme.mit.inf.dslreasoner.run/src/hu/bme/mit/inf/dslreasoner/run/script/MeasurementScriptRunner.xtend | |||
@@ -0,0 +1,351 @@ | |||
1 | package hu.bme.mit.inf.dslreasoner.run.script | ||
2 | |||
3 | import com.google.gson.Gson | ||
4 | import hu.bme.mit.inf.dslreasoner.ecore2logic.EClassMapper | ||
5 | import hu.bme.mit.inf.dslreasoner.ecore2logic.Ecore2Logic | ||
6 | import hu.bme.mit.inf.dslreasoner.ecore2logic.Ecore2LogicConfiguration | ||
7 | import hu.bme.mit.inf.dslreasoner.ecore2logic.Ecore2Logic_Trace | ||
8 | import hu.bme.mit.inf.dslreasoner.logic.model.builder.DocumentationLevel | ||
9 | import hu.bme.mit.inf.dslreasoner.logic.model.builder.TypeScopes | ||
10 | import hu.bme.mit.inf.dslreasoner.logic.model.logiclanguage.DefinedElement | ||
11 | import hu.bme.mit.inf.dslreasoner.logic.model.logiclanguage.IntLiteral | ||
12 | import hu.bme.mit.inf.dslreasoner.logic.model.logiclanguage.RealLiteral | ||
13 | import hu.bme.mit.inf.dslreasoner.logic.model.logiclanguage.StringLiteral | ||
14 | import hu.bme.mit.inf.dslreasoner.logic.model.logiclanguage.Type | ||
15 | import hu.bme.mit.inf.dslreasoner.logic.model.logiclanguage.TypeDefinition | ||
16 | import hu.bme.mit.inf.dslreasoner.logic.model.logicproblem.LogicProblem | ||
17 | import hu.bme.mit.inf.dslreasoner.logic.model.logicresult.IntStatisticEntry | ||
18 | import hu.bme.mit.inf.dslreasoner.logic.model.logicresult.LogicresultFactory | ||
19 | import hu.bme.mit.inf.dslreasoner.logic.model.logicresult.ModelResult | ||
20 | import hu.bme.mit.inf.dslreasoner.logic.model.logicresult.RealStatisticEntry | ||
21 | import hu.bme.mit.inf.dslreasoner.logic.model.logicresult.Statistics | ||
22 | import hu.bme.mit.inf.dslreasoner.logic.model.logicresult.StringStatisticEntry | ||
23 | import hu.bme.mit.inf.dslreasoner.logic2ecore.Logic2Ecore | ||
24 | import hu.bme.mit.inf.dslreasoner.run.EcoreLoader | ||
25 | import hu.bme.mit.inf.dslreasoner.run.FAMLoader | ||
26 | import hu.bme.mit.inf.dslreasoner.run.FileSystemLoader | ||
27 | import hu.bme.mit.inf.dslreasoner.run.MetamodelLoader | ||
28 | import hu.bme.mit.inf.dslreasoner.run.SatelliteLoader | ||
29 | import hu.bme.mit.inf.dslreasoner.run.YakinduLoader | ||
30 | import hu.bme.mit.inf.dslreasoner.util.CollectionsUtil | ||
31 | import hu.bme.mit.inf.dslreasoner.viatra2logic.Viatra2Logic | ||
32 | import hu.bme.mit.inf.dslreasoner.viatra2logic.Viatra2LogicConfiguration | ||
33 | import hu.bme.mit.inf.dslreasoner.viatrasolver.logic2viatra.cardinality.PolyhedralScopePropagatorConstraints | ||
34 | import hu.bme.mit.inf.dslreasoner.viatrasolver.logic2viatra.cardinality.PolyhedralScopePropagatorSolver | ||
35 | import hu.bme.mit.inf.dslreasoner.viatrasolver.logic2viatra.cardinality.ScopePropagatorStrategy | ||
36 | import hu.bme.mit.inf.dslreasoner.viatrasolver.partialinterpretation2logic.InstanceModel2Logic | ||
37 | import hu.bme.mit.inf.dslreasoner.viatrasolver.partialinterpretationlanguage.partial2logicannotations.PartialModelRelation2Assertion | ||
38 | import hu.bme.mit.inf.dslreasoner.viatrasolver.reasoner.ViatraReasoner | ||
39 | import hu.bme.mit.inf.dslreasoner.viatrasolver.reasoner.ViatraReasonerConfiguration | ||
40 | import hu.bme.mit.inf.dslreasoner.workspace.FileSystemWorkspace | ||
41 | import hu.bme.mit.inf.dslreasoner.workspace.ReasonerWorkspace | ||
42 | import java.io.FileReader | ||
43 | import java.util.HashMap | ||
44 | import java.util.HashSet | ||
45 | import java.util.Map | ||
46 | import java.util.Set | ||
47 | import org.eclipse.emf.ecore.EObject | ||
48 | import org.eclipse.emf.ecore.resource.Resource | ||
49 | import org.eclipse.emf.ecore.xmi.impl.XMIResourceFactoryImpl | ||
50 | import org.eclipse.viatra.query.patternlanguage.emf.EMFPatternLanguageStandaloneSetup | ||
51 | import org.eclipse.viatra.query.runtime.api.ViatraQueryEngineOptions | ||
52 | import org.eclipse.viatra.query.runtime.rete.matcher.ReteBackendFactory | ||
53 | import org.eclipse.xtend.lib.annotations.Data | ||
54 | |||
55 | class MeasurementScriptRunner { | ||
56 | static val MODEL_SIZE_GAP = 0 | ||
57 | static val SCOPE_PROPAGATOR_TIMEOUT = 10 | ||
58 | static val USEC_TO_MSEC = 1000000 | ||
59 | |||
60 | static extension val LogicresultFactory = LogicresultFactory.eINSTANCE | ||
61 | |||
62 | val MeasurementScript script | ||
63 | val ReasonerWorkspace inputWorkspace | ||
64 | val ReasonerWorkspace outputWorkspace | ||
65 | val MetamodelLoader metamodelLoader | ||
66 | |||
67 | new(MeasurementScript script) { | ||
68 | this.script = script | ||
69 | inputWorkspace = new FileSystemWorkspace(script.inputPath + "/", "") | ||
70 | outputWorkspace = new FileSystemWorkspace(script.outputPath + | ||
71 | "/", '''«script.domain»_«script.solver»_«script.scope»_«script.scopePropagator ?: "na"»_«script.propagatedConstraints ?: "na"»_«script.polyhedronSolver ?: "na"»_''') | ||
72 | metamodelLoader = switch (script.domain) { | ||
73 | case fs: new FileSystemLoader(inputWorkspace) | ||
74 | case ecore: new EcoreLoader(inputWorkspace) | ||
75 | case Yakindu: new YakinduLoader(inputWorkspace) => [useSynchronization = false; useComplexStates = true] | ||
76 | case FAM: new FAMLoader(inputWorkspace) | ||
77 | case satellite: new SatelliteLoader(inputWorkspace) | ||
78 | default: throw new IllegalArgumentException("Unsupported domain: " + script.domain) | ||
79 | } | ||
80 | } | ||
81 | |||
82 | def run() { | ||
83 | if (script.sizes.empty) { | ||
84 | return | ||
85 | } | ||
86 | val start = System.currentTimeMillis | ||
87 | val warmupSize = script.sizes.head | ||
88 | for (var int i = 0; i < script.warmupIterations; i++) { | ||
89 | System.err.println('''Warmup «i + 1»/«script.warmupIterations»...''') | ||
90 | runExperiment(warmupSize) | ||
91 | } | ||
92 | val warmupEnd = System.currentTimeMillis | ||
93 | System.err.println('''Warmup completed in «(warmupEnd - start) / 1000» seconds''') | ||
94 | for (size : script.sizes) { | ||
95 | var int failures = 0 | ||
96 | for (var int i = 0; i < script.iterations; i++) { | ||
97 | System.err.println("Running GC...") | ||
98 | runGc() | ||
99 | System.err.println('''Iteration «i + 1»/«script.iterations» of size «size»...''') | ||
100 | val startTime = System.currentTimeMillis | ||
101 | val result = runExperiment(size) | ||
102 | val headerPrefix = '''«script.toCsvHeader»,«size»,«i + 1»,«result.resultName»''' | ||
103 | println('''«headerPrefix»,startTime,«startTime»''') | ||
104 | println('''«headerPrefix»,logic2SolverTransformationTime,«result.statistics.transformationTime»''') | ||
105 | println('''«headerPrefix»,solverTime,«result.statistics.solverTime»''') | ||
106 | for (statistic : result.statistics.entries) { | ||
107 | val valueString = switch (statistic) { | ||
108 | IntStatisticEntry: statistic.value.toString | ||
109 | RealStatisticEntry: statistic.value.toString | ||
110 | StringStatisticEntry: statistic.value.toString | ||
111 | default: statistic.toString | ||
112 | } | ||
113 | println('''«headerPrefix»,«statistic.name»,«valueString»''') | ||
114 | } | ||
115 | if (script.saveModels && result.model !== null) { | ||
116 | outputWorkspace.writeModel(result.model, '''«size»_«i + 1».xmi''') | ||
117 | } | ||
118 | if (result.resultName === "InsuficientResourcesResultImpl") { | ||
119 | failures++ | ||
120 | } | ||
121 | System.out.flush | ||
122 | } | ||
123 | if (failures == script.iterations) { | ||
124 | System.err.println("All measurements failed") | ||
125 | return | ||
126 | } | ||
127 | } | ||
128 | val end = System.currentTimeMillis | ||
129 | System.err.println('''Measurement completed in «(end - start) / 1000» seconds''') | ||
130 | } | ||
131 | |||
132 | private static def void runGc() { | ||
133 | System.gc | ||
134 | Thread.sleep(100) | ||
135 | System.gc | ||
136 | Thread.sleep(100) | ||
137 | System.gc | ||
138 | Thread.sleep(800) | ||
139 | } | ||
140 | |||
141 | private def runExperiment(int modelSize) { | ||
142 | if (script.solver != Solver.ViatraSolver) { | ||
143 | throw new IllegalArgumentException("Only VIATRA-Generator is supported") | ||
144 | } | ||
145 | val config = new ViatraReasonerConfiguration | ||
146 | config.solutionScope.numberOfRequiredSolutions = 1 | ||
147 | config.scopePropagatorStrategy = switch (script.scopePropagator) { | ||
148 | case none: | ||
149 | ScopePropagatorStrategy.None | ||
150 | case basic: | ||
151 | switch (script.propagatedConstraints) { | ||
152 | case none: | ||
153 | ScopePropagatorStrategy.Basic | ||
154 | case typeHierarchy: | ||
155 | ScopePropagatorStrategy.BasicTypeHierarchy | ||
156 | case relations, | ||
157 | case hints: | ||
158 | throw new IllegalArgumentException( | ||
159 | "Basic scope propagator does not support relational and hint constraints") | ||
160 | default: | ||
161 | throw new IllegalArgumentException("Unknown scope constraints: " + script.propagatedConstraints) | ||
162 | } | ||
163 | case polyhedral: { | ||
164 | val constraints = switch (script.propagatedConstraints) { | ||
165 | case none: | ||
166 | throw new IllegalArgumentException( | ||
167 | "Polyhedral scope propagator needs at least type hierarchy constraints") | ||
168 | case typeHierarchy: | ||
169 | PolyhedralScopePropagatorConstraints.TypeHierarchy | ||
170 | case relations, | ||
171 | case hints: | ||
172 | PolyhedralScopePropagatorConstraints.Relational | ||
173 | default: | ||
174 | throw new IllegalArgumentException("Unknown scope constraints: " + script.propagatedConstraints) | ||
175 | } | ||
176 | val polyhedronSolver = switch (script.polyhedronSolver) { | ||
177 | case Z3Integer: PolyhedralScopePropagatorSolver.Z3Integer | ||
178 | case Z3Real: PolyhedralScopePropagatorSolver.Z3Real | ||
179 | case Cbc: PolyhedralScopePropagatorSolver.Cbc | ||
180 | case Clp: PolyhedralScopePropagatorSolver.Clp | ||
181 | default: throw new IllegalArgumentException("Unknown polyhedron solver: " + script.polyhedronSolver) | ||
182 | } | ||
183 | val updateHeuristic = script.scopeHeuristic != ScopeHeuristic.basic | ||
184 | new ScopePropagatorStrategy.Polyhedral(constraints, polyhedronSolver, updateHeuristic, | ||
185 | SCOPE_PROPAGATOR_TIMEOUT) | ||
186 | } | ||
187 | default: | ||
188 | throw new IllegalArgumentException("Unknown scope propagator: " + script.scopePropagator) | ||
189 | } | ||
190 | config.runtimeLimit = script.timeout | ||
191 | config.documentationLevel = if(script.saveTemporaryFiles) DocumentationLevel.NORMAL else DocumentationLevel.NONE | ||
192 | config.debugConfiguration.partialInterpretatioVisualiser = null | ||
193 | config.searchSpaceConstraints.additionalGlobalConstraints += metamodelLoader.additionalConstraints | ||
194 | |||
195 | val modelLoadingStart = System.nanoTime | ||
196 | val metamodelDescriptor = metamodelLoader.loadMetamodel | ||
197 | val partialModelDescriptor = metamodelLoader.loadPartialModel | ||
198 | val queryDescriptor = metamodelLoader.loadQueries(metamodelDescriptor) | ||
199 | val modelLoadingTime = System.nanoTime - modelLoadingStart | ||
200 | |||
201 | val domain2LogicTransformationStart = System.nanoTime | ||
202 | val Ecore2Logic ecore2Logic = new Ecore2Logic | ||
203 | val Viatra2Logic viatra2Logic = new Viatra2Logic(ecore2Logic) | ||
204 | val InstanceModel2Logic instanceModel2Logic = new InstanceModel2Logic | ||
205 | var modelGeneration = ecore2Logic.transformMetamodel(metamodelDescriptor, new Ecore2LogicConfiguration()) | ||
206 | var problem = modelGeneration.output | ||
207 | problem = instanceModel2Logic.transform( | ||
208 | modelGeneration, | ||
209 | partialModelDescriptor | ||
210 | ).output | ||
211 | problem = viatra2Logic.transformQueries( | ||
212 | queryDescriptor, | ||
213 | modelGeneration, | ||
214 | new Viatra2LogicConfiguration | ||
215 | ).output | ||
216 | initializeScope(config, modelSize, problem, ecore2Logic, modelGeneration.trace) | ||
217 | if (script.propagatedConstraints == ScopeConstraints.hints) { | ||
218 | config.hints = metamodelLoader.getHints(ecore2Logic, modelGeneration.trace) | ||
219 | } | ||
220 | val domain2LogicTransformationTime = System.nanoTime - domain2LogicTransformationStart | ||
221 | |||
222 | if (config.documentationLevel != DocumentationLevel.NONE) { | ||
223 | outputWorkspace.writeModel(problem, "initial.logicproblem") | ||
224 | } | ||
225 | |||
226 | val solver = new ViatraReasoner | ||
227 | val result = solver.solve(problem, config, outputWorkspace) | ||
228 | val statistics = result.statistics | ||
229 | statistics.entries += createIntStatisticEntry => [ | ||
230 | name = "modelLoadingTime" | ||
231 | value = (modelLoadingTime / USEC_TO_MSEC) as int | ||
232 | ] | ||
233 | statistics.entries += createIntStatisticEntry => [ | ||
234 | name = "domain2LogicTransformationTime" | ||
235 | value = (domain2LogicTransformationTime / USEC_TO_MSEC) as int | ||
236 | ] | ||
237 | var EObject modelResult = null | ||
238 | if (result instanceof ModelResult) { | ||
239 | val intepretations = solver.getInterpretations(result) | ||
240 | if (intepretations.size != 1) { | ||
241 | throw new IllegalStateException("Expected 1 interpretation, got " + intepretations.size) | ||
242 | } | ||
243 | var resultTransformationStart = System.nanoTime | ||
244 | val logic2Ecore = new Logic2Ecore(ecore2Logic) | ||
245 | modelResult = logic2Ecore.transformInterpretation(intepretations.head, modelGeneration.trace) | ||
246 | val resultTransformationTime = System.nanoTime - resultTransformationStart | ||
247 | statistics.entries += createIntStatisticEntry => [ | ||
248 | name = "ecore2LogicTransformationTime" | ||
249 | value = (resultTransformationTime / USEC_TO_MSEC) as int | ||
250 | ] | ||
251 | } | ||
252 | |||
253 | new ExperimentResult(result.class.simpleName, statistics, modelResult) | ||
254 | } | ||
255 | |||
256 | private def initializeScope(ViatraReasonerConfiguration config, int modelSize, LogicProblem problem, | ||
257 | EClassMapper eClassMapper, Ecore2Logic_Trace trace) { | ||
258 | val knownElements = initializeKnownElements(problem, config.typeScopes) | ||
259 | if (modelSize < 0) { | ||
260 | config.typeScopes.minNewElements = 0 | ||
261 | config.typeScopes.maxNewElements = TypeScopes.Unlimited | ||
262 | } else { | ||
263 | val numberOfKnownElements = knownElements.values.flatten.toSet.size | ||
264 | val newElementCount = modelSize - numberOfKnownElements | ||
265 | config.typeScopes.minNewElements = newElementCount | ||
266 | config.typeScopes.maxNewElements = newElementCount + MODEL_SIZE_GAP | ||
267 | } | ||
268 | switch (script.scope) { | ||
269 | case none: | ||
270 | return | ||
271 | case quantiles: { | ||
272 | val quantiles = metamodelLoader.typeQuantiles | ||
273 | for (eClassInScope : eClassMapper.allClassesInScope(trace)) { | ||
274 | val quantile = quantiles.get(eClassInScope.name) | ||
275 | if (quantile !== null) { | ||
276 | val type = eClassMapper.TypeofEClass(trace, eClassInScope) | ||
277 | val knownInstances = knownElements.get(type) | ||
278 | val currentCount = if(knownInstances === null) 0 else knownInstances.size | ||
279 | val lowCount = Math.floor(modelSize * quantile.low) as int | ||
280 | val highCount = Math.ceil((modelSize + MODEL_SIZE_GAP) * quantile.high) as int | ||
281 | config.typeScopes.minNewElementsByType.put(type, lowCount - currentCount) | ||
282 | config.typeScopes.maxNewElementsByType.put(type, highCount - currentCount) | ||
283 | } | ||
284 | } | ||
285 | } | ||
286 | default: | ||
287 | throw new IllegalArgumentException("Unknown scope: " + script.scope) | ||
288 | } | ||
289 | } | ||
290 | |||
291 | /* | ||
292 | * Copied from hu.bme.mit.inf.dslreasoner.application.execution.ScopeLoader.initialiseknownElements(LogicProblem, TypeScopes) | ||
293 | */ | ||
294 | private static def initializeKnownElements(LogicProblem p, TypeScopes s) { | ||
295 | val Map<Type, Set<DefinedElement>> res = new HashMap | ||
296 | |||
297 | // 1. fill map with every types | ||
298 | for (t : p.types) { | ||
299 | res.put(t, new HashSet) | ||
300 | } | ||
301 | |||
302 | // 2. fill map with every objects | ||
303 | for (definedType : p.types.filter(TypeDefinition)) { | ||
304 | val supertypes = CollectionsUtil.<Type>transitiveClosureStar(definedType)[supertypes] | ||
305 | for (supertype : supertypes) { | ||
306 | for (element : definedType.elements) { | ||
307 | res.get(supertype).add(element) | ||
308 | } | ||
309 | } | ||
310 | } | ||
311 | val partialModelContents = p.annotations.filter(PartialModelRelation2Assertion).map[target].toList.map [ | ||
312 | eAllContents.toIterable | ||
313 | ].flatten.toList | ||
314 | s.knownIntegers += partialModelContents.filter(IntLiteral).map[it.value] | ||
315 | s.knownReals += partialModelContents.filter(RealLiteral).map[it.value] | ||
316 | s.knownStrings += partialModelContents.filter(StringLiteral).map[it.value] | ||
317 | |||
318 | res | ||
319 | } | ||
320 | |||
321 | public static def void main(String[] args) { | ||
322 | if (args.length != 1) { | ||
323 | System.err.println("Missing measurement script name.") | ||
324 | System.exit(-1) | ||
325 | } | ||
326 | EMFPatternLanguageStandaloneSetup.doSetup | ||
327 | ViatraQueryEngineOptions.setSystemDefaultBackends(ReteBackendFactory.INSTANCE, ReteBackendFactory.INSTANCE, | ||
328 | ReteBackendFactory.INSTANCE) | ||
329 | Resource.Factory.Registry.INSTANCE.getExtensionToFactoryMap().put("*", new XMIResourceFactoryImpl) | ||
330 | val config = readConfig(args.get(0)) | ||
331 | val runnner = new MeasurementScriptRunner(config) | ||
332 | runnner.run() | ||
333 | } | ||
334 | |||
335 | static def readConfig(String scriptPath) { | ||
336 | val gson = new Gson | ||
337 | val reader = new FileReader(scriptPath) | ||
338 | try { | ||
339 | gson.fromJson(reader, MeasurementScript) | ||
340 | } finally { | ||
341 | reader.close | ||
342 | } | ||
343 | } | ||
344 | |||
345 | @Data | ||
346 | private static class ExperimentResult { | ||
347 | String resultName | ||
348 | Statistics statistics | ||
349 | EObject model | ||
350 | } | ||
351 | } | ||