001package org.dllearner.server; 002 003import org.apache.commons.collections15.BidiMap; 004import org.apache.commons.collections15.bidimap.DualHashBidiMap; 005import org.dllearner.algorithms.DisjointClassesLearner; 006import org.dllearner.algorithms.SimpleSubclassLearner; 007import org.dllearner.algorithms.celoe.CELOE; 008import org.dllearner.algorithms.properties.*; 009import org.dllearner.configuration.spring.editors.ConfigHelper; 010import org.dllearner.core.*; 011import org.dllearner.kb.SparqlEndpointKS; 012import org.dllearner.kb.sparql.SPARQLTasks; 013import org.dllearner.kb.sparql.SparqlEndpoint; 014import org.dllearner.kb.sparql.SparqlKnowledgeSource; 015import org.dllearner.accuracymethods.AccMethodFMeasure; 016import org.dllearner.learningproblems.AxiomScore; 017import org.dllearner.learningproblems.ClassLearningProblem; 018import org.dllearner.reasoning.ClosedWorldReasoner; 019import org.dllearner.reasoning.SPARQLReasoner; 020import org.dllearner.utilities.Helper; 021import org.dllearner.utilities.datastructures.SortedSetTuple; 022import org.dllearner.utilities.examples.AutomaticNegativeExampleFinderSPARQL2; 023import org.dllearner.utilities.owl.ManchesterOWLSyntaxOWLObjectRendererImplExt; 024import org.dllearner.utilities.owl.OWL2SPARULConverter; 025import org.json.JSONArray; 026import org.json.simple.JSONObject; 027import org.semanticweb.owlapi.apibinding.OWLManager; 028import org.semanticweb.owlapi.io.OWLObjectRenderer; 029import org.semanticweb.owlapi.model.*; 030import uk.ac.manchester.cs.owl.owlapi.OWLClassImpl; 031import uk.ac.manchester.cs.owl.owlapi.OWLDataPropertyImpl; 032import uk.ac.manchester.cs.owl.owlapi.OWLObjectPropertyImpl; 033 034import javax.servlet.ServletException; 035import javax.servlet.http.HttpServlet; 036import javax.servlet.http.HttpServletRequest; 037import javax.servlet.http.HttpServletResponse; 038import java.io.IOException; 039import java.io.PrintWriter; 040import java.net.SocketTimeoutException; 041import java.net.URL; 042import java.util.*; 043import java.util.concurrent.*; 044 045public class EnrichmentServlet extends HttpServlet { 046 047 private static List<Class<? extends LearningAlgorithm>> objectPropertyAlgorithms; 048 private static List<Class<? extends LearningAlgorithm>> dataPropertyAlgorithms; 049 private static List<Class<? extends LearningAlgorithm>> classAlgorithms; 050 private static BidiMap<AxiomType, Class<? extends LearningAlgorithm>> axiomType2Class; 051 052 private static final List<String> entityTypes = Arrays.asList("class", "objectproperty", "dataproperty"); 053 054 private static String validAxiomTypes = ""; 055 056 private OWL2SPARULConverter sparul; 057 private OWLOntology ont; 058 059 static { 060 axiomType2Class = new DualHashBidiMap<>(); 061 axiomType2Class.put(AxiomType.SUBCLASS_OF, SimpleSubclassLearner.class); 062 axiomType2Class.put(AxiomType.EQUIVALENT_CLASSES, CELOE.class); 063 axiomType2Class.put(AxiomType.DISJOINT_CLASSES, DisjointClassesLearner.class); 064 axiomType2Class.put(AxiomType.SUB_OBJECT_PROPERTY, SubObjectPropertyOfAxiomLearner.class); 065 axiomType2Class.put(AxiomType.EQUIVALENT_OBJECT_PROPERTIES, EquivalentObjectPropertyAxiomLearner.class); 066 axiomType2Class.put(AxiomType.DISJOINT_OBJECT_PROPERTIES, DisjointObjectPropertyAxiomLearner.class); 067 axiomType2Class.put(AxiomType.OBJECT_PROPERTY_DOMAIN, ObjectPropertyDomainAxiomLearner.class); 068 axiomType2Class.put(AxiomType.OBJECT_PROPERTY_RANGE, ObjectPropertyRangeAxiomLearner.class); 069 axiomType2Class.put(AxiomType.FUNCTIONAL_OBJECT_PROPERTY, FunctionalObjectPropertyAxiomLearner.class); 070 axiomType2Class.put(AxiomType.INVERSE_FUNCTIONAL_OBJECT_PROPERTY, 071 InverseFunctionalObjectPropertyAxiomLearner.class); 072 axiomType2Class.put(AxiomType.REFLEXIVE_OBJECT_PROPERTY, ReflexiveObjectPropertyAxiomLearner.class); 073 axiomType2Class.put(AxiomType.IRREFLEXIVE_OBJECT_PROPERTY, IrreflexiveObjectPropertyAxiomLearner.class); 074 axiomType2Class.put(AxiomType.SYMMETRIC_OBJECT_PROPERTY, SymmetricObjectPropertyAxiomLearner.class); 075 axiomType2Class.put(AxiomType.ASYMMETRIC_OBJECT_PROPERTY, AsymmetricObjectPropertyAxiomLearner.class); 076 axiomType2Class.put(AxiomType.TRANSITIVE_OBJECT_PROPERTY, TransitiveObjectPropertyAxiomLearner.class); 077 axiomType2Class.put(AxiomType.SUB_DATA_PROPERTY, SubDataPropertyOfAxiomLearner.class); 078 axiomType2Class.put(AxiomType.EQUIVALENT_DATA_PROPERTIES, EquivalentDataPropertyAxiomLearner.class); 079 axiomType2Class.put(AxiomType.DISJOINT_DATA_PROPERTIES, DisjointDataPropertyAxiomLearner.class); 080 axiomType2Class.put(AxiomType.DATA_PROPERTY_DOMAIN, DataPropertyDomainAxiomLearner.class); 081 axiomType2Class.put(AxiomType.DATA_PROPERTY_RANGE, DataPropertyRangeAxiomLearner.class); 082 axiomType2Class.put(AxiomType.FUNCTIONAL_DATA_PROPERTY, FunctionalDataPropertyAxiomLearner.class); 083 084 objectPropertyAlgorithms = new LinkedList<>(); 085 objectPropertyAlgorithms.add(DisjointObjectPropertyAxiomLearner.class); 086 objectPropertyAlgorithms.add(EquivalentObjectPropertyAxiomLearner.class); 087 objectPropertyAlgorithms.add(SubObjectPropertyOfAxiomLearner.class); 088 objectPropertyAlgorithms.add(ObjectPropertyDomainAxiomLearner.class); 089 objectPropertyAlgorithms.add(ObjectPropertyRangeAxiomLearner.class); 090 objectPropertyAlgorithms.add(FunctionalObjectPropertyAxiomLearner.class); 091 objectPropertyAlgorithms.add(InverseFunctionalObjectPropertyAxiomLearner.class); 092 objectPropertyAlgorithms.add(SymmetricObjectPropertyAxiomLearner.class); 093 objectPropertyAlgorithms.add(AsymmetricObjectPropertyAxiomLearner.class); 094 objectPropertyAlgorithms.add(TransitiveObjectPropertyAxiomLearner.class); 095 objectPropertyAlgorithms.add(ReflexiveObjectPropertyAxiomLearner.class); 096 objectPropertyAlgorithms.add(IrreflexiveObjectPropertyAxiomLearner.class); 097 098 dataPropertyAlgorithms = new LinkedList<>(); 099 dataPropertyAlgorithms.add(DisjointDataPropertyAxiomLearner.class); 100 dataPropertyAlgorithms.add(EquivalentDataPropertyAxiomLearner.class); 101 dataPropertyAlgorithms.add(FunctionalDataPropertyAxiomLearner.class); 102 dataPropertyAlgorithms.add(DataPropertyDomainAxiomLearner.class); 103 dataPropertyAlgorithms.add(DataPropertyRangeAxiomLearner.class); 104 dataPropertyAlgorithms.add(SubDataPropertyOfAxiomLearner.class); 105 106 classAlgorithms = new LinkedList<>(); 107 classAlgorithms.add(DisjointClassesLearner.class); 108 classAlgorithms.add(SimpleSubclassLearner.class); 109 classAlgorithms.add(CELOE.class); 110 111 for (AxiomType type : AxiomType.AXIOM_TYPES) { 112 validAxiomTypes += type.getName() + ", "; 113 } 114 } 115 116 private static final int DEFAULT_MAX_EXECUTION_TIME_IN_SECONDS = 10; 117 private static final int DEFAULT_MAX_NR_OF_RETURNED_AXIOMS = 10; 118 private static final double DEFAULT_THRESHOLD = 0.75; 119 120 private String cacheDir; 121 122 private OWLDataFactory dataFactory; 123 124 public EnrichmentServlet() { 125 OWLOntologyManager man = OWLManager.createOWLOntologyManager(); 126 OWLOntology ont = null; 127 try { 128 ont = man.createOntology(); 129 } catch (OWLOntologyCreationException e1) { 130 e1.printStackTrace(); 131 } 132 sparul = new OWL2SPARULConverter(ont, false); 133 dataFactory = man.getOWLDataFactory(); 134 } 135 136 @Override 137 public void init() throws ServletException { 138 super.init(); 139 cacheDir = getServletContext().getRealPath("cache"); 140 } 141 142 @Override 143 protected void doGet(HttpServletRequest req, HttpServletResponse resp) throws ServletException, IOException { 144 long timeStamp = System.currentTimeMillis(); 145 String endpointURL = req.getParameter("endpoint_url"); 146 if (endpointURL == null) { 147 throw new IllegalStateException("Missing parameter: endpoint"); 148 } 149 String graphURI = req.getParameter("default_graph_uri"); 150 151 SparqlEndpoint endpoint = new SparqlEndpoint(new URL(endpointURL), Collections.singletonList(graphURI), 152 Collections.<String> emptyList()); 153 154 final boolean useInference = req.getParameter("use_inference") == null ? false : Boolean.valueOf(req 155 .getParameter("use_inference")); 156 157 final int maxNrOfReturnedAxioms = req.getParameter("max_returned_axioms") == null ? DEFAULT_MAX_NR_OF_RETURNED_AXIOMS : Integer.parseInt(req.getParameter("max_returned_axioms")); 158 final int maxExecutionTimeInSeconds = req.getParameter("max_execution_time") == null ? DEFAULT_MAX_EXECUTION_TIME_IN_SECONDS : Integer.parseInt(req.getParameter("max_execution_time")); 159 final double threshold = req.getParameter("threshold") == null ? DEFAULT_THRESHOLD : Double.parseDouble(req.getParameter("threshold")); 160 161 String resourceURI = req.getParameter("resource_uri"); 162 if (resourceURI == null) { 163 throw new IllegalStateException("Missing parameter: resource_uri"); 164 } 165 166 String axiomTypeStrings[] = req.getParameterValues("axiom_types"); 167 if (axiomTypeStrings == null) { 168 throw new IllegalStateException("Missing parameter: axiom_types"); 169 } 170 axiomTypeStrings = axiomTypeStrings[0].split(","); 171 172 Collection<AxiomType> requestedAxiomTypes = new HashSet<>(); 173 for (String typeStr : axiomTypeStrings) { 174 AxiomType type = AxiomType.getAxiomType(typeStr.trim()); 175 if (type == null) { 176 throw new IllegalStateException("Illegal axiom type: " + typeStr + ". Please use one of " + validAxiomTypes); 177 } else { 178 requestedAxiomTypes.add(type); 179 } 180 } 181 182 SPARQLTasks st = new SPARQLTasks(endpoint); 183 String entityType = req.getParameter("entity_type"); 184 final OWLEntity entity; 185 if(entityType != null){ 186 if(oneOf(entityType, entityTypes)){ 187 entity = getEntity(resourceURI, entityType, endpoint); 188 } else { 189 throw new IllegalStateException("Illegal entity type: " + entityType + ". Please use one of " + entityTypes); 190 } 191 192 } else { 193 entity = st.guessResourceType(resourceURI, true); 194 } 195 196 Collection<AxiomType> executableAxiomTypes = new HashSet<>(); 197 Collection<AxiomType> omittedAxiomTypes = new HashSet<>(); 198 Collection<AxiomType<? extends OWLAxiom>> possibleAxiomTypes = AxiomAlgorithms.getAxiomTypes(entity.getEntityType()); 199 for(AxiomType type : requestedAxiomTypes){ 200 if(possibleAxiomTypes.contains(type)){ 201 executableAxiomTypes.add(type); 202 } else { 203 omittedAxiomTypes.add(type); 204 } 205 } 206 207 final SparqlEndpointKS ks = new SparqlEndpointKS(endpoint); 208 try { 209 ks.init(); 210 } catch (ComponentInitException e) { 211 e.printStackTrace(); 212 } 213 // check if endpoint supports SPARQL 1.1 214 boolean supportsSPARQL_1_1 = st.supportsSPARQL_1_1(); 215 ks.setSupportsSPARQL_1_1(supportsSPARQL_1_1); 216 217 final SPARQLReasoner reasoner = new SPARQLReasoner(new SparqlEndpointKS(endpoint)); 218 if (useInference && !reasoner.isPrepared()) { 219 System.out.print("Precomputing subsumption hierarchy ... "); 220 long startTime = System.currentTimeMillis(); 221 reasoner.prepareSubsumptionHierarchy(); 222 System.out.println("done in " + (System.currentTimeMillis() - startTime) + " ms"); 223 } 224 225 JSONArray result = new JSONArray(); 226 227 ExecutorService executor = Executors.newFixedThreadPool(Runtime.getRuntime().availableProcessors()); 228 List<Future<JSONObject>> list = new ArrayList<>(); 229 230 final OWLObjectRenderer renderer = new ManchesterOWLSyntaxOWLObjectRendererImplExt(); 231// renderer.setShortFormProvider(new ManchesterOWLSyntaxPrefixNameShortFormProvider(new DefaultPrefixManager())); 232 233 234 235 for (final AxiomType axiomType : executableAxiomTypes) { 236 Callable<JSONObject> worker = new Callable<JSONObject>() { 237 238 @Override 239 public JSONObject call() throws Exception { 240 JSONObject result = new JSONObject(); 241 JSONArray axiomArray = new JSONArray(); 242 List<EvaluatedAxiom> axioms = getEvaluatedAxioms(ks, reasoner, entity, axiomType, maxExecutionTimeInSeconds, threshold, maxNrOfReturnedAxioms, useInference); 243 for(EvaluatedAxiom ax : axioms){ 244 JSONObject axiomObject = new JSONObject(); 245 OWLAxiom axiom = ax.getAxiom(); 246 axiomObject.put("axiom", axiom); 247 axiomObject.put("axiom_rendered", renderer.render(axiom)); 248 axiomObject.put("axiom_sparul", getSPARUL(axiom)); 249 axiomObject.put("confidence", ax.getScore().getAccuracy()); 250 axiomArray.put(axiomObject); 251 } 252 result.put("axiom_type", axiomType); 253 result.put("axioms", axiomArray); 254 return result; 255 } 256 257 }; 258 Future<JSONObject> submit = executor.submit(worker); 259 list.add(submit); 260 } 261 262 263 for (Future<JSONObject> future : list) { 264 try { 265 JSONObject array = future.get(); 266 result.put(array); 267 } catch (InterruptedException | ExecutionException e) { 268 e.printStackTrace(); 269 } 270 } 271 272 executor.shutdown(); 273 274 resp.setContentType("application/json"); 275 PrintWriter pw = resp.getWriter(); 276 JSONObject finalResult = new JSONObject(); 277 finalResult.put("result", result); 278 finalResult.put("timestamp", timeStamp); 279 finalResult.put("execution time", System.currentTimeMillis()-timeStamp); 280 finalResult.put("endpoint url", endpointURL); 281 finalResult.put("graph", graphURI); 282 finalResult.put("resource uri", resourceURI); 283 finalResult.put("entity type", entityType); 284 finalResult.put("omitted axiom types", omittedAxiomTypes); 285 String resultString = finalResult.toJSONString(); 286 if(req.getParameter("jsonp_callback") != null){ 287 resultString = req.getParameter("jsonp_callback") + "(" + resultString + ")"; 288 } 289 pw.print(resultString); 290 pw.close(); 291 } 292 293 private String getSPARUL(OWLAxiom axiom){ 294 return sparul.convert(new AddAxiom(ont, axiom)); 295 } 296 297 private boolean oneOf(String value, String... possibleValues){ 298 for(String v : possibleValues){ 299 if(v.equals(value)){ 300 return true; 301 } 302 } 303 return false; 304 } 305 306 private boolean oneOf(String value, Collection<String> possibleValues){ 307 for(String v : possibleValues){ 308 if(v.equals(value)){ 309 return true; 310 } 311 } 312 return false; 313 } 314 315 private List<EvaluatedAxiom> getEvaluatedAxioms(SparqlEndpointKS endpoint, SPARQLReasoner reasoner, 316 OWLEntity entity, AxiomType axiomType, int maxExecutionTimeInSeconds, 317 double threshold, int maxNrOfReturnedAxioms, boolean useInference) { 318 List<EvaluatedAxiom> learnedAxioms = new ArrayList<>(); 319 try { 320 learnedAxioms = applyLearningAlgorithm(axiomType2Class.get(axiomType), endpoint, reasoner, entity, maxExecutionTimeInSeconds, threshold, maxNrOfReturnedAxioms); 321 } catch (ComponentInitException e) { 322 e.printStackTrace(); 323 } 324 return learnedAxioms; 325 } 326 327 private List<EvaluatedAxiom> applyLearningAlgorithm(Class<? extends LearningAlgorithm> algorithmClass, 328 SparqlEndpointKS ks, SPARQLReasoner reasoner, OWLEntity entity, int maxExecutionTimeInSeconds, double threshold, int maxNrOfReturnedAxioms) 329 throws ComponentInitException { 330 List<EvaluatedAxiom> learnedAxioms = null; 331 if(algorithmClass == CELOE.class){ 332 learnedAxioms = applyCELOE(ks, entity.asOWLClass(), true, false, threshold); 333 } else { 334 AxiomLearningAlgorithm learner = null; 335 try { 336 337 learner = (AxiomLearningAlgorithm) algorithmClass.getConstructor(SparqlEndpointKS.class).newInstance(ks); 338 } catch (Exception e) { 339 e.printStackTrace(); 340 } 341 if (classAlgorithms.contains(algorithmClass)) { 342 ConfigHelper.configure(learner, "classToDescribe", entity); 343 } else { 344 ConfigHelper.configure(learner, "propertyToDescribe", entity); 345 } 346 ConfigHelper.configure(learner, "maxExecutionTimeInSeconds", maxExecutionTimeInSeconds); 347 // if(reasoner != null){ 348 ((AbstractAxiomLearningAlgorithm) learner).setReasoner(reasoner); 349 // } 350 learner.init(); 351 String algName = AnnComponentManager.getName(learner); 352 System.out.print("Applying " + algName + " on " + entity + " ... "); 353 long startTime = System.currentTimeMillis(); 354 try { 355 learner.start(); 356 } catch (Exception e) { 357 if (e.getCause() instanceof SocketTimeoutException) { 358 System.out.println("Query timed out (endpoint possibly too slow)."); 359 } else { 360 e.printStackTrace(); 361 } 362 } 363 long runtime = System.currentTimeMillis() - startTime; 364 System.out.println("done in " + runtime + " ms"); 365 learnedAxioms = learner.getCurrentlyBestEvaluatedAxioms(maxNrOfReturnedAxioms, threshold); 366 } 367 368 return learnedAxioms; 369 } 370 371 private List<EvaluatedAxiom> applyCELOE(SparqlEndpointKS ks, OWLClass nc, boolean equivalence, boolean reuseKnowledgeSource, double threshold) throws ComponentInitException { 372 373 // get instances of class as positive examples 374 SPARQLReasoner sr = new SPARQLReasoner(ks); 375 SortedSet<OWLIndividual> posExamples = sr.getIndividuals(nc, 20); 376 if(posExamples.isEmpty()){ 377 System.out.println("Skipping CELOE because class " + nc.toString() + " is empty."); 378 return Collections.emptyList(); 379 } 380 SortedSet<String> posExStr = Helper.getStringSet(posExamples); 381 382 // use own implementation of negative example finder 383 long startTime = System.currentTimeMillis(); 384 System.out.print("finding negatives ... "); 385 AutomaticNegativeExampleFinderSPARQL2 finder = new AutomaticNegativeExampleFinderSPARQL2(ks.getEndpoint()); 386 SortedSet<OWLIndividual> negExamples = finder.getNegativeExamples(nc, posExamples, 20); 387 SortedSetTuple<OWLIndividual> examples = new SortedSetTuple<>(posExamples, negExamples); 388 long runTime = System.currentTimeMillis() - startTime; 389 System.out.println("done (" + negExamples.size()+ " examples fround in " + runTime + " ms)"); 390 391 SparqlKnowledgeSource ks2; 392 AbstractReasonerComponent rc; 393 ks2 = new SparqlKnowledgeSource(); 394 ks2.setInstances(Helper.getStringSet(examples.getCompleteSet())); 395 ks2.setUrl(ks.getEndpoint().getURL()); 396 ks2.setDefaultGraphURIs(new TreeSet<>(ks.getEndpoint().getDefaultGraphURIs())); 397 ks2.setUseLits(false); 398 ks2.setUseCacheDatabase(true); 399 ks2.setCacheDir(cacheDir); 400 ks2.setRecursionDepth(2); 401 ks2.setCloseAfterRecursion(true); 402 ks2.setDissolveBlankNodes(false); 403 ks2.setSaveExtractedFragment(true); 404 startTime = System.currentTimeMillis(); 405 System.out.print("getting knowledge base fragment ... "); 406 ks2.init(); 407 runTime = System.currentTimeMillis() - startTime; 408 System.out.println("done in " + runTime + " ms"); 409 rc = new ClosedWorldReasoner(ks2); 410 rc.init(); 411 412 ClassLearningProblem lp = new ClassLearningProblem(rc); 413 lp.setClassToDescribe(nc); 414 lp.setEquivalence(equivalence); 415 lp.setAccuracyMethod(new AccMethodFMeasure(true)); 416 lp.setMaxExecutionTimeInSeconds(10); 417 lp.init(); 418 419 CELOE la = new CELOE(lp, rc); 420 la.setMaxExecutionTimeInSeconds(10); 421 la.setNoisePercentage(25); 422 la.init(); 423 startTime = System.currentTimeMillis(); 424 System.out.print("running CELOE (for " + (equivalence ? "equivalent classes" : "sub classes") + ") ... "); 425 la.start(); 426 runTime = System.currentTimeMillis() - startTime; 427 System.out.println("done in " + runTime + " ms"); 428 429 // convert the result to axioms (to make it compatible with the other algorithms) 430 List<? extends EvaluatedDescription<? extends Score>> learnedDescriptions = la.getCurrentlyBestEvaluatedDescriptions(threshold); 431 List<EvaluatedAxiom> learnedAxioms = new LinkedList<>(); 432 for(EvaluatedDescription<? extends Score> learnedDescription : learnedDescriptions) { 433 OWLAxiom axiom; 434 if(equivalence) { 435 axiom = dataFactory.getOWLEquivalentClassesAxiom(nc, learnedDescription.getDescription()); 436 } else { 437 axiom = dataFactory.getOWLSubClassOfAxiom(nc, learnedDescription.getDescription()); 438 } 439 Score score = lp.computeScore(learnedDescription.getDescription()); 440 learnedAxioms.add(new EvaluatedAxiom(axiom, new AxiomScore(score.getAccuracy()))); 441 } 442 return learnedAxioms; 443 } 444 445 private OWLEntity getEntity(String resourceURI, String entityType, SparqlEndpoint endpoint) { 446 OWLEntity entity = null; 447 switch (entityType) { 448 case "class": 449 entity = new OWLClassImpl(IRI.create(resourceURI)); 450 break; 451 case "objectproperty": 452 entity = new OWLObjectPropertyImpl(IRI.create(resourceURI)); 453 break; 454 case "dataproperty": 455 entity = new OWLDataPropertyImpl(IRI.create(resourceURI)); 456 break; 457 default: 458 SPARQLTasks st = new SPARQLTasks(endpoint); 459 entity = st.guessResourceType(resourceURI, true); 460 break; 461 } 462 return entity; 463 } 464}