001/* 002 * To change this license header, choose License Headers in Project Properties. 003 * To change this template file, choose Tools | Templates 004 * and open the template in the editor. 005 */ 006package org.dllearner.algorithms.probabilistic.structure.distributed.unife.leap; 007 008import com.clarkparsia.pellet.owlapiv3.PelletReasoner; 009import com.clarkparsia.pellet.owlapiv3.PelletReasonerFactory; 010import com.google.common.collect.Sets; 011import java.io.File; 012import java.math.BigDecimal; 013import java.util.ArrayList; 014import java.util.Collections; 015import java.util.HashSet; 016import java.util.LinkedHashSet; 017import java.util.List; 018import java.util.Map; 019import java.util.NavigableSet; 020import java.util.Set; 021import java.util.TreeSet; 022import mpi.Intracomm; 023import mpi.MPI; 024import mpi.MPIException; 025import org.apache.log4j.Logger; 026import org.dllearner.core.AbstractClassExpressionLearningProblem; 027import org.dllearner.core.ComponentAnn; 028import org.dllearner.core.ComponentInitException; 029import org.dllearner.core.EvaluatedDescription; 030import org.dllearner.core.LearningProblemUnsupportedException; 031import org.dllearner.core.config.ConfigOption; 032import org.dllearner.learningproblems.ClassLearningProblem; 033import org.dllearner.learningproblems.PosNegLP; 034import org.dllearner.learningproblems.PosOnlyLP; 035import org.dllearner.core.probabilistic.unife.AbstractPSLA; 036import org.dllearner.core.probabilistic.unife.ParameterLearningException; 037import org.dllearner.core.probabilistic.unife.StructureLearningException; 038import org.dllearner.core.probabilistic.distributed.unife.AbstractEDGEDistributed; 039import org.dllearner.core.probabilistic.distributed.unife.CommunicatorGroupNotAssignedException; 040import org.dllearner.core.probabilistic.distributed.unife.DistributedComponent; 041import org.dllearner.utils.unife.OWLUtils; 042import org.dllearner.utils.unife.ReflectionHelper; 043import org.semanticweb.owlapi.apibinding.OWLManager; 044import org.semanticweb.owlapi.model.AxiomType; 045import org.semanticweb.owlapi.model.IRI; 046import org.semanticweb.owlapi.model.OWLAnnotation; 047import org.semanticweb.owlapi.model.OWLAxiom; 048import org.semanticweb.owlapi.model.OWLClass; 049import org.semanticweb.owlapi.model.OWLClassExpression; 050import org.semanticweb.owlapi.model.OWLDataFactory; 051import org.semanticweb.owlapi.model.OWLIndividual; 052import org.semanticweb.owlapi.model.OWLOntology; 053import org.semanticweb.owlapi.model.OWLOntologyCreationException; 054import org.semanticweb.owlapi.model.OWLOntologyManager; 055import org.semanticweb.owlapi.model.OWLOntologyStorageException; 056import org.semanticweb.owlapi.model.OWLSubClassOfAxiom; 057import org.springframework.beans.factory.annotation.Autowired; 058import unife.bundle.exception.InconsistencyException; 059import unife.bundle.utilities.BundleUtilities; 060import static unife.edge.mpi.EDGEMPIConstants.*; 061import unife.edge.mpi.MPIUtilities; 062import unife.edge.utilities.EDGEUtilities; 063import unife.math.utilities.MathUtilities; 064 065/** 066 * 067 * @author Giuseppe Cota <giuseppe.cota@unife.it>, Riccardo Zese 068 * <riccardo.zese@unife.it> 069 */ 070@ComponentAnn(name = "LEAPDistributed", shortName = "leapdistr", version = 1.0) 071public class LEAPDistributed extends AbstractPSLA implements DistributedComponent { 072 073 @ConfigOption(description = "stop difference between log-likelihood of two consecutive iterations", defaultValue = "0.00001") 074 private BigDecimal differenceLL = MathUtilities.getBigDecimal(0.00001, 5); 075 076 private BigDecimal currentDifferenceLL; 077 078 @ConfigOption(description = "stop ratio between log-likelihood of two consecutive iterations", defaultValue = "0.00001") 079 private BigDecimal ratioLL = MathUtilities.getBigDecimal(0.00001, 5); 080 081 private BigDecimal currentRatioLL; 082 083 @ConfigOption(description = "maximum number of iterations", defaultValue = "2147000000") 084 private long maxIterations = 2147000000L; 085 086 private long currentIteration = 0; 087 088 @ConfigOption(description = "accuracy used during the computation of the probabilistic values (number of digital places)", defaultValue = "5") 089 private int accuracy = 5; 090 091 private static final Logger logger = Logger.getLogger(LEAPDistributed.class.getName()); 092 093 @ConfigOption(description = "probabilistic target axioms which can be deleted from the ontology") 094 private String targetAxiomsFilename; 095 096 @ConfigOption(defaultValue = "owl:learnedClass", description = "You can specify a start class for the algorithm. To do this, you have to use Manchester OWL syntax without using prefixes.") 097 private OWLClass dummyClass; 098 099 @ConfigOption(description = "number of mpi processes of probabilistic structure learning algorithm", defaultValue = "1") 100 private int procPSLA = 1; 101 102 @ConfigOption(description = "number of mpi processes of parameter learning algorithm for each probabilistic structure learner process", defaultValue = "1") 103 private int procPLA = 1; 104 105 private int revisionBeamDim = 10; 106 107 private AbstractEDGEDistributed edge; 108 109 private TreeSet<Revision> beamRevisions = new TreeSet<>(); 110 111 private Revision bestRevision; 112 private Revision previousBestRevision = new Revision(); 113 private List<Revision> revisions; 114 private List<Set<Revision>> revisionsDistribution; 115 116 private final Object countRevisionsLock = new Object(); 117 private int countRevisions = 0; 118 119 private LinkedHashSet<OWLAxiom> targetAxioms = new LinkedHashSet<>(); 120 121 private int myRank; 122 private int structureLearnerRank; 123 private int parameterLearnerRank; 124 private Intracomm structureLearnerComm; 125 private Intracomm parameterLearnerComm; 126 private OWLOntology originalOntology; 127 private final int UPDATE = 10; 128 private final int REMOVE = 11; 129 130 @Override 131 public void init() throws ComponentInitException { 132 logger.debug("Start init() LEAPDistributed"); 133 currentIteration = 0; 134 OWLOntologyManager manager = OWLManager.createOWLOntologyManager(); 135 // create dummy class 136 if (dummyClass == null) { 137 dummyClass = manager.getOWLDataFactory().getOWLClass(IRI.create("owl:learnedClass")); 138 } 139 140 // read the ontology conatining the target axioms 141 logger.debug("read the ontology containing the target axioms"); 142 if (targetAxiomsFilename != null) { 143 try { 144 OWLOntology targetAxiomsOntology = manager.loadOntologyFromOntologyDocument(new File(targetAxiomsFilename)); 145 for (OWLAxiom targetAxiom : EDGEUtilities.get_ax_filtered(targetAxiomsOntology)) { 146 targetAxioms.add(targetAxiom); 147 } 148 } catch (OWLOntologyCreationException ex) { 149 logger.error("Cannot get the target probabilistic axioms."); 150 throw new ComponentInitException(ex); 151 } 152 } 153 // get rank 154 try { 155 myRank = MPI.COMM_WORLD.getRank(); 156 } catch (MPIException mpiEX) { 157 logger.error("Cannot get the rank of the process"); 158 throw new ComponentInitException(mpiEX); 159 } 160 // create groups and communicators 161 int mpiProcesses; 162 logger.debug(myRank + " - create groups and communicators"); 163 try { 164 mpiProcesses = MPI.COMM_WORLD.getSize(); 165 if (mpiProcesses != procPSLA * procPLA) { 166 String msg = myRank + " - The number of process must be (procPSLA * procPLA): " 167 + (procPSLA * procPLA) + " instead there are " 168 + mpiProcesses + " processes."; 169 logger.error(msg); 170 throw new ComponentInitException(msg); 171 } 172 } catch (MPIException mpiEx) { 173 logger.error(myRank + " - Cannot get the number of processes"); 174 throw new ComponentInitException(mpiEx); 175 } 176 /* Determine row and column position */ 177 int row = myRank / procPLA; 178 int col = myRank % procPLA; 179 logger.debug(myRank + " - Parameter Learner Group " 180 + " Group id: " + row 181 + " Process Rank: " + col); 182 try { 183 // The processes in the same row belong to the same parameter learner 184 // group communicator 185 //Intracomm comm = MPI.COMM_WORLD.dup(); 186 parameterLearnerComm = MPI.COMM_WORLD.split(row, col); 187// parameterLearnerComm = MPI.COMM_WORLD.split(row, col); 188 parameterLearnerRank = parameterLearnerComm.getRank(); 189 logger.debug(myRank + " - Parameter Learner Group created." 190 + " Group id: " + row 191 + " Process Rank: " + parameterLearnerRank); 192 // The processes in the first column belong to the structure learner 193 // group communicator (There can only be one structure learner 194 // group communicator ) 195 logger.debug("test a"); 196 structureLearnerComm = MPI.COMM_WORLD.split(col, row); 197 logger.debug("test b"); 198 structureLearnerRank = structureLearnerComm.getRank(); 199 logger.debug("test c"); 200 if (col != 0) { 201 structureLearnerComm = null; 202 } else { 203 logger.debug(myRank + " - Structure Learner Group" 204 + " Group id: " + col 205 + " Process Rank: " + row); 206 logger.debug(myRank + " - Structure Learner Group created. Rank: " 207 + " Group id: " + col 208 + " Process Rank: " + structureLearnerRank); 209 } 210 211 } catch (MPIException mpiEx) { 212 logger.error(myRank + " - Cannot create the group communicators"); 213 throw new ComponentInitException(mpiEx); 214 } 215 216 // these few lines are used to force the use of SHOIN DL, 217 // i.e. without qualified cardinality restriction 218// if (cela instanceof CELOE) { 219// CELOE celoe = (CELOE) cela; 220// LengthLimitedRefinementOperator operator = celoe.getOperator(); 221// if (operator instanceof RhoDRDown) { 222// LengthLimitedRefinementOperator newOperator = new RhoDRDown((RhoDRDown) operator); 223// // force no qualified cardinality restriction, we use SHOIN 224// ((RhoDRDown) newOperator).setUseCardinalityRestrictions(false); 225// 226// newOperator.init(); 227// celoe.setOperator(newOperator); 228// } 229// } 230 logger.debug(myRank + " - getting the individuals"); 231 Set<OWLIndividual> positiveIndividuals; 232 Set<OWLIndividual> negativeIndividuals; 233 AbstractClassExpressionLearningProblem lp = cela.getLearningProblem(); 234 if (lp instanceof PosNegLP) { 235 positiveIndividuals = ((PosNegLP) lp).getPositiveExamples(); 236 negativeIndividuals = ((PosNegLP) lp).getNegativeExamples(); 237 } else if (lp instanceof PosOnlyLP) { 238 positiveIndividuals = ((PosOnlyLP) lp).getPositiveExamples(); 239 // use pseudo-negative individuals 240 negativeIndividuals = Sets.difference(lp.getReasoner().getIndividuals(), positiveIndividuals); 241 } else if (lp instanceof ClassLearningProblem) { 242 // Java Reflection has been used to get values from private fields. 243 //It's neither a conventional way nor the universally suggested idea, 244 // but in this case is the only way to extract positive and negative individuals 245 // without modifing the DLLearner code (the creation of a plugin is the objective) 246 try { 247 List<OWLIndividual> positiveIndividualsList = ReflectionHelper.getPrivateField(lp, "classInstances"); 248 positiveIndividuals = new TreeSet<>(positiveIndividualsList); 249 negativeIndividuals = new TreeSet<>((List<OWLIndividual>) ReflectionHelper.getPrivateField(lp, "superClassInstances")); 250 } catch (NoSuchFieldException | IllegalArgumentException | IllegalAccessException e) { 251 String msg = myRank + " - Cannot extract the individuals from" 252 + " learning problem: " + e.getMessage(); 253 logger.error(msg); 254 throw new ComponentInitException(msg); 255 } 256 257 } else { 258 try { 259 throw new LearningProblemUnsupportedException(lp.getClass(), this.getClass()); 260 } catch (LearningProblemUnsupportedException e) { 261 throw new ComponentInitException(e.getMessage()); 262 } 263 } 264 // convert the individuals into assertional axioms 265 logger.debug(myRank + " - convert the individuals into assertional axioms"); 266 OWLDataFactory owlFactory = manager.getOWLDataFactory(); 267 Set<OWLAxiom> positiveExamples = new HashSet<>(); 268 for (OWLIndividual ind : positiveIndividuals) { 269 OWLAxiom axiom = owlFactory.getOWLClassAssertionAxiom(dummyClass, ind); 270 positiveExamples.add(axiom); 271 } 272 273 Set<OWLAxiom> negativeExamples = new HashSet<>(); 274 for (OWLIndividual ind : negativeIndividuals) { 275 OWLAxiom axiom = owlFactory.getOWLClassAssertionAxiom(dummyClass, ind); 276 negativeExamples.add(axiom); 277 } 278 279 //AbstractEDGEDistributed edge = (AbstractEDGEDistributed) pla; 280 edge.setPositiveExampleAxioms(positiveExamples); 281 edge.setNegativeExampleAxioms(negativeExamples); 282 283 } 284 285 @Override 286 public void start() { 287 stop = false; 288 isRunning = true; 289 currentIteration = 0; 290 long totalTimeMills = System.currentTimeMillis(); 291 long celaTimeMills = 0; 292 try { 293 originalOntology = BundleUtilities.copyOntology(edge.getSourcesOntology()); 294 } catch (OWLOntologyCreationException e) { 295 logger.error(myRank + "Error: " + e.getMessage()); 296 throw new StructureLearningException(e); 297 } 298 //AbstractEDGEDistributed edge = (AbstractEDGEDistributed) pla; 299 // First step: run Distributed EDGE 300// try { 301// edge.changeSourcesOntology(BundleUtilities.copyOntology(originalOntology)); 302// } catch (OWLOntologyCreationException e) { 303// logger.error("morte", e); 304// System.exit(-1); 305// } 306 edge.start(); 307 308 logger.debug(myRank + " - First EDGE cycle terminated."); 309 logger.debug(myRank + " - Initial Log-likelihood: " + edge.getLL()); 310 //OWLOntology originalOntology = edge.getLearnedOntology(); 311 312 if (structureLearnerComm != null) { 313 logger.debug(myRank + " - Structure Learner"); 314 if (MPIUtilities.isMaster(structureLearnerComm)) { 315 logger.debug(myRank + " - Structure Learner Master"); 316 List<Boolean> boolVars = new ArrayList<>(); 317 for (int i = 0; i < targetAxioms.size(); i++) { 318 boolVars.add(true); 319 } 320 Revision startRevision = new Revision(targetAxioms, boolVars, new LinkedHashSet<OWLSubClassOfAxiom>(), edge.getLL()); 321 beamRevisions.add(startRevision); 322 bestRevision = startRevision; 323 currentDifferenceLL = bestRevision.getLL().subtract(previousBestRevision.getLL()); 324 currentRatioLL = currentDifferenceLL.divide(previousBestRevision.getLL(), accuracy, BigDecimal.ROUND_HALF_UP); 325 do { 326 //BigDecimal CLL0 = MathUtilities.getBigDecimal(-2.2 * Math.pow(10, 10), edge.getAccuracy()); 327 //BigDecimal CLL1 = edge.getLL(); 328 329 // generate all the refinements 330 //refinements = generateRefinements(); 331 // divido i raffinamenti tra i vari slaves 332 // TO DO 333 // receive refinement 334 // TO DO 335 // ***start temporary code lines*** 336 Revision revision = startRevision; 337 // *** end *** 338 339 OWLOntology ontology = generateOntologyFromRevision(originalOntology, revision); 340 // send the refinement to the EDGE slaves 341 try { 342 MPIUtilities.sendBCastSignal(START, parameterLearnerComm); 343 logger.debug(myRank + " - Sent START signal to EDGE slaves"); 344 int sentBytes = MPIUtilities.sendBCastObject(revision, parameterLearnerComm); 345 logger.debug(myRank + " - Sent revision to " 346 + MPIUtilities.getSlaves(parameterLearnerComm) + " slaves " 347 + "(" + sentBytes + " bytes)"); 348 } catch (MPIException mpiEx) { 349 logger.error(myRank + " - Cannot send to EDGE slaves the refinement: " + mpiEx.getMessage()); 350 throw new StructureLearningException(mpiEx); 351 } 352 353// Set<KnowledgeSource> newSources = Collections.singleton((KnowledgeSource) new OWLAPIOntology(ontology)); 354// AbstractReasonerComponent reasoner = cela.getReasoner(); 355// reasoner.changeSources(newSources); 356// try { 357// reasoner.init(); 358// cela.init(); 359// } catch (ComponentInitException cie) { 360// logger.error(myRank + " - Error: " + cie.getMessage()); 361// throw new StructureLearningException(cie); 362// } 363 // start class expression learning algorithm 364 celaTimeMills = System.currentTimeMillis(); 365 cela.start(); 366 celaTimeMills = System.currentTimeMillis() - celaTimeMills; 367 // get the best class expressions 368 NavigableSet<? extends EvaluatedDescription> evaluatedDescriptions = cela.getCurrentlyBestEvaluatedDescriptions(); 369 // convert the class expressions into axioms 370 OWLOntologyManager manager = OWLManager.createOWLOntologyManager(); 371 LinkedHashSet<OWLSubClassOfAxiom> candidateAxioms = convertIntoAxioms(manager, evaluatedDescriptions); 372 373 logger.debug(myRank + " - current best revision LL: " + bestRevision.getLL()); 374 // perform a greedy search 375 logger.debug(myRank + " - Start greedy search"); 376 // temporaneo i raffinamenti dopo dovranno essere assegnati ad ogni processo 377 Revision revisionResult = greedySearch(ontology, startRevision, candidateAxioms); 378 logger.debug(myRank + " - Greedy search finished"); 379 // aggiungerlo eventualmente alla lista beam 380 // TO DO 381 // aggiornare il best 382 // TO DO 383 // ***righe temporanee*** 384 logger.debug(myRank + " - current LL: " + revisionResult.getLL()); 385 logger.debug(myRank + " - current best revision LL: " + bestRevision.getLL()); 386 if (revisionResult.getLL().compareTo(bestRevision.getLL()) > 0) { 387 logger.info(myRank + " - Found a better revision with LL: " + revisionResult.getLL()); 388 previousBestRevision = bestRevision; 389 bestRevision = revisionResult; 390 } 391 // ***fine righe temporanee*** 392 // update criteria 393 updateTerminationCriteria(); 394 } while (!terminationCriteriaSatisfied()); 395 try { 396 MPIUtilities.sendBCastSignal(TERMINATE, parameterLearnerComm); 397 } catch (MPIException e) { 398 logger.error("Cannot send TERMINATE signal to EDGE slaves"); 399 throw new StructureLearningException(e); 400 } 401 OWLOntology finalOntology = edge.getSourcesOntology(); 402 // In case replace super class 403 if (cela.getLearningProblem() instanceof ClassLearningProblem) { 404 finalOntology = replaceSuperClass(finalOntology, bestRevision.getLearnedAxioms()); 405 } else { 406 for (OWLAxiom axiom : bestRevision.getLearnedAxioms()) { 407 logger.info(myRank + " - Learned Axiom: " + axiom); 408 } 409 } 410 // final step save the ontology 411 try { 412 OWLUtils.saveOntology(finalOntology, outputFile, outFormat); 413 } catch (OWLOntologyStorageException e) { 414 String msg = myRank + " - Cannot save the learned ontology: " + e.getMessage(); 415 throw new StructureLearningException(msg); 416 } 417 totalTimeMills = System.currentTimeMillis() - totalTimeMills; 418 printTimings(totalTimeMills, celaTimeMills, edge.getTimeMap()); 419 420 } else { // structure learner slaves 421 // Leggo i raffinamenti 422 // TO DO 423 // trovo le class expression 424 // invio in broadcast al parameter learner communicator group 425 // apprendo i parametri 426 throw new UnsupportedOperationException("Not supported yet!"); 427 } 428 } else if (parameterLearnerComm != null) { // EDGE slaves 429 // devo solo eseguire degli EDGE distribuiti 430 // prendo la lista di boolean e costruisco una nuova ontologia 431 logger.debug(myRank + " - Parameter Learner Slave"); 432 boolean terminate = false; 433 while (!terminate) { 434 int signal; 435 try { 436 signal = MPIUtilities.recvBCastSignal(MASTER, parameterLearnerComm); 437 } catch (MPIException mpiEx) { 438 String msg = myRank + " - Cannot receive synchronization signal " + mpiEx.getMessage(); 439 logger.error(msg); 440 throw new StructureLearningException(mpiEx); 441 } 442 if (signal == START) { 443 // read the refinement and modify the ontology 444 logger.debug(myRank + " - Received START signal."); 445 logger.debug(myRank + " - Waiting to receive ontology revision."); 446 OWLOntology ontology; 447 try { 448 Revision revision = (Revision) MPIUtilities.recvBCastObject(MASTER, parameterLearnerComm); 449 ontology = generateOntologyFromRevision(originalOntology, revision); 450 } catch (MPIException mpiEx) { 451 String msg = myRank + " - Cannot receive Refinement from EDGE Master: " + mpiEx.getMessage(); 452 logger.error(msg); 453 throw new StructureLearningException(mpiEx); 454 } 455 boolean stopEDGE = false; 456 // receive signal 457 try { 458 signal = MPIUtilities.recvBCastSignal(MASTER, parameterLearnerComm); 459 } catch (MPIException e) { 460 String msg = myRank + " - Cannot receive signal: " + e.getMessage(); 461 logger.error(msg); 462 throw new StructureLearningException(msg); 463 } 464 while (signal == START) { 465 edge.reset(); 466 // add axiom 467 logger.debug(myRank + " - Waiting for axiom to add"); 468 OWLAxiom addedAxiom = recvAddAxiom(ontology); 469 edge.changeSourcesOntology(ontology); 470 if (edge.getSourcesOntology().containsAxiom(addedAxiom)) { 471 logger.debug(myRank + " - Axiom added into the ontology"); 472 } else { 473 String msg = myRank + " - Impossible to add the received axiom"; 474 logger.debug(msg); 475 throw new StructureLearningException(msg); 476 } 477 // compute edge 478 edge.start(); 479 // waiting for decision: keep the axiom or remove it? 480 481 try { 482 signal = MPIUtilities.recvBCastSignal(MASTER, parameterLearnerComm); 483 } catch (MPIException e) { 484 String msg = myRank + " - Cannot receive signal: " + e.getMessage(); 485 logger.error(msg); 486 throw new StructureLearningException(msg); 487 } 488 if (signal == UPDATE) { // keep the added axiom 489 logger.debug(myRank + " - Received UPDATE ontology signal"); 490 updateOntology(); 491 } else if (signal == REMOVE) { // remove the added axiom 492 logger.debug(myRank + " - Received REMOVE axiom from ontology signal"); 493 removeAxiom(ontology, addedAxiom); 494 } else { // wrong signal 495 String msg = myRank + " - Wrong signal received: " + signal; 496 logger.error(msg); 497 throw new StructureLearningException(msg); 498 } 499 // receive signal 500 try { 501 signal = MPIUtilities.recvBCastSignal(MASTER, parameterLearnerComm); 502 logger.debug(myRank + " - Received " + signal + " from MASTER"); 503 } catch (MPIException e) { 504 String msg = myRank + " - Cannot receive signal: " + e.getMessage(); 505 logger.error(msg); 506 throw new StructureLearningException(msg); 507 } 508 509 } 510 logger.debug(myRank + " - Received STOP signal."); 511 } else if (signal == TERMINATE) { 512 logger.debug(myRank + " - Received TERMINATE signal."); 513 terminate = true; 514 } 515 } 516 517 } else { 518 // ERROR 519 String msg = myRank + " - MPI process not assigned to any communicator group"; 520 logger.error(msg); 521 throw new CommunicatorGroupNotAssignedException(msg); 522 } 523 isRunning = false; 524 } 525 526 /** 527 * @return the targetAxiomsFilename 528 */ 529 public String getTargetAxiomsFilename() { 530 return targetAxiomsFilename; 531 } 532 533 /** 534 * @param targetAxiomsFilename the targetAxiomsFilename to set 535 */ 536 public void setTargetAxiomsFilename(String targetAxiomsFilename) { 537 this.targetAxiomsFilename = targetAxiomsFilename; 538 } 539 540 /** 541 * @return the PSLAproc 542 */ 543 public int getProcPSLA() { 544 return procPSLA; 545 } 546 547 /** 548 * @param procPSLA the PSLAproc to set 549 */ 550 public void setProcPSLA(int procPSLA) { 551 this.procPSLA = procPSLA; 552 } 553 554 /** 555 * @return the PLAproc 556 */ 557 public int getProcPLA() { 558 return procPLA; 559 } 560 561 /** 562 * @param procPLA the procPLA to set 563 */ 564 public void setProcPLA(int procPLA) { 565 this.procPLA = procPLA; 566 } 567 568 /** 569 * @return the differenceLL 570 */ 571 public BigDecimal getDifferenceLL() { 572 return differenceLL; 573 } 574 575 public void setDifferenceLL(double differenceLL) { 576 this.differenceLL = MathUtilities.getBigDecimal(differenceLL, accuracy); 577 } 578 579 /** 580 * @return the ratioLL 581 */ 582 public BigDecimal getRatioLL() { 583 return ratioLL; 584 } 585 586 public void setRatioLL(double ratioLL) { 587 this.ratioLL = MathUtilities.getBigDecimal(ratioLL, accuracy); 588 } 589 590 /** 591 * @return the maxIterations 592 */ 593 public long getMaxIterations() { 594 return maxIterations; 595 } 596 597 /** 598 * @param maxIterations the maxIterations to set 599 */ 600 public void setMaxIterations(long maxIterations) { 601 this.maxIterations = maxIterations; 602 } 603 604 /** 605 * @return the accuracy 606 */ 607 public int getAccuracy() { 608 return accuracy; 609 } 610 611 /** 612 * @param accuracy the accuracy to set 613 */ 614 public void setAccuracy(int accuracy) { 615 this.accuracy = accuracy; 616 } 617 618 private List<Revision> generateRevisions() { 619 // see http://stackoverflow.com/questions/10923601/java-generator-of-trues-falses-combinations-by-giving-the-number-n 620 // for generating refinements 621 // takes all the target axioms and the best refinement from the beam 622 // and generate all the refinements 623 throw new UnsupportedOperationException("Not supported yet."); //To change body of generated methods, choose Tools | Templates. 624 } 625 626 /** 627 * @return the dummyClass 628 */ 629 public OWLClass getDummyClass() { 630 return dummyClass; 631 } 632 633 /** 634 * @param dummyClass the dummyClass to set 635 */ 636 public void setDummyClass(OWLClass dummyClass) { 637 this.dummyClass = dummyClass; 638 } 639 640 private LinkedHashSet<OWLSubClassOfAxiom> convertIntoAxioms(OWLOntologyManager manager, NavigableSet<? extends EvaluatedDescription> evaluatedDescriptions) { 641 LinkedHashSet<OWLSubClassOfAxiom> axioms = new LinkedHashSet<>(evaluatedDescriptions.size()); 642 OWLDataFactory factory = manager.getOWLDataFactory(); 643 for (EvaluatedDescription description : evaluatedDescriptions.descendingSet()) { 644 OWLAnnotation annotation = factory. 645 getOWLAnnotation(BundleUtilities.PROBABILISTIC_ANNOTATION_PROPERTY, factory.getOWLLiteral(description.getAccuracy())); 646 OWLSubClassOfAxiom axiom = factory. 647 getOWLSubClassOfAxiom((OWLClassExpression) description.getDescription(), dummyClass, Collections.singleton(annotation)); 648 axioms.add(axiom); 649 } 650 return axioms; 651 } 652 653 /** 654 * This class implements a thread that will be in listening and when a slave 655 * is ready will send it a refinement. 656 */ 657// class RefinementsListener implements Runnable { 658// 659// private BigDecimal[] probs; 660// private int sentRefinements; 661// int recvRefinementsEval = 0; 662// 663// public RefinementsListener(int sentRefinements) { 664// super(); 665// this.sentRefinements = sentRefinements; 666// } 667// 668// @Override 669// public void run() { 670// try { 671// boolean stopRequest = false; 672// Refinement refinement = null; 673// int slaveId = 0; 674// while (sentRefinements != recvRefinementsEval) { 675// 676// synchronized (countRefinementsLock) { 677// logger.debug(myRank + " - T_2"); 678// if (refinement != null) { 679// refinementsDistribution.get(slaveId).add(refinement); 680// } 681// refinement = nextRefinement(); 682// } 683// Status recvStat = structureLearnerComm.probe(MPI.ANY_SOURCE, MPI.ANY_TAG); 684// byte[] buffer = new byte[recvStat.getCount(MPI.BYTE)]; 685// logger.debug(myRank + " - Waiting for a refinement result..."); 686// // receive 687// structureLearnerComm.recv(buffer, buffer.length, MPI.BYTE, MPI.ANY_SOURCE, MPI.ANY_TAG); 688// recvRefinementsEval++; 689// // get example probability 690// byte[] exProbBuff = new byte[buffer.length - (Integer.SIZE / 8)]; 691// int pos = MPI.COMM_WORLD.unpack(buffer, 0, exProbBuff, exProbBuff.length, MPI.BYTE); 692// int indexEx = recvStat.getTag() - 1; 693// slaveId = recvStat.getSource(); 694// probs[indexEx] = (BigDecimal) MPIUtilities.byteArrayToObject(exProbBuff); 695// String msg = myRank + " - RECV from " + recvStat.getSource() 696// + " - " + BundleUtilities.getManchesterSyntaxString(initialExamples.get(indexEx).getKey()) 697// + " - prob: " + probs[indexEx] 698// + " - tag: " + recvStat.getTag(); 699// if (!test) { 700// int[] numBoolVars = new int[1]; 701// MPI.COMM_WORLD.unpack(buffer, pos, numBoolVars, 1, MPI.INT); 702// msg += " - #vars: " + numBoolVars[0]; 703// // indices of the axioms used in the BDD 704// int[] axiomsIdxs = new int[numBoolVars[0]]; 705// if (numBoolVars[0] > 0) { 706// // receive the buffer containing the indices of the used axioms 707// MPI.COMM_WORLD.recv(axiomsIdxs, axiomsIdxs.length, MPI.INT, slaveId, recvStat.getTag()); 708// } 709// 710// for (int j = 0; j < numBoolVars[0]; j++) { 711// usedAxioms[axiomsIdxs[j]] = true; 712// } 713// } 714// logger.debug(msg); 715// // I've received the query/example results, 716// // now it's time to send another example 717// if (example == null) { 718// if (!stopRequest) { 719// // Send stop request to all the slaves 720// new Thread() { 721// @Override 722// public void run() { 723// synchronized (countActuallySentExamplesLock) { 724// while (sentRefinements != countActuallySentExamples) { 725// try { 726// countActuallySentExamplesLock.wait(); 727// } catch (InterruptedException ex) { 728// logger.fatal(myRank + " - Error: " + ex.getMessage()); 729// throw new RuntimeException(ex); 730// } 731// } 732// } 733// logger.debug(myRank + " - Sending stop signal to all slaves"); 734// try { 735// MPIUtilities.sendSignal(ALL, STOP, MPI.COMM_WORLD); 736// } catch (MPIException mpiEx) { 737// logger.error(myRank + " - Error: " + mpiEx.getMessage()); 738// throw new RuntimeException(mpiEx); 739// } 740// } 741// }.start(); 742// stopRequest = true; 743// } 744// } else { 745// // send Example to the slave 746// if (slaveId == 0) { 747// throw new RuntimeException("Inconsistent slave receiver!"); 748// } 749// sentRefinements++; 750// (new Thread(new EDGEMPIDynamic.ExampleSender(slaveId, example, sentRefinements))).start(); 751// 752// } 753// } 754// if (!stopRequest) { 755// logger.debug(myRank + " - Sending stop signal to all slaves"); 756// MPIUtilities.sendSignal(ALL, STOP, MPI.COMM_WORLD); 757// stopRequest = true; // unused 758// } 759// } catch (MPIException mpiEx) { 760// logger.error(myRank + " - Error: " + mpiEx.getMessage()); 761// throw new RuntimeException(mpiEx); 762// } catch (Exception ex) { 763// logger.error(myRank + " - Error: " + ex.getMessage()); 764// throw new RuntimeException(ex); 765// } 766// } 767// 768// /** 769// * @param sentRefinements the sentExamples to set 770// */ 771// public void setSentRefinements(int sentRefinements) { 772// this.sentRefinements = sentRefinements; 773// } 774// 775// } 776 private Revision nextRefinement() { 777 if (countRevisions < revisions.size()) { 778 countRevisions++; 779 return revisions.get(countRevisions - 1); 780 } else { 781 return null; 782 } 783 } 784 785 private Revision greedySearch(OWLOntology ontology, Revision revision, LinkedHashSet<OWLSubClassOfAxiom> candidateAxioms) { 786 BigDecimal LL0 = edge.getLL(); // da rivedere 787 logger.debug(myRank + " - Resetting EDGE"); 788 edge.reset(); 789 edge.changeSourcesOntology(ontology); 790 LinkedHashSet<OWLSubClassOfAxiom> learnedAxioms = new LinkedHashSet<>(); 791 OWLDataFactory df = OWLManager.getOWLDataFactory(); 792 for (OWLSubClassOfAxiom axiom : candidateAxioms) { 793 try { 794// logger.debug(myRank + " - Sending START signal to parameter learner slaves"); 795// logger.debug(myRank + " - Signal sent to the slaves"); 796 logger.debug(myRank + " - Adding Axiom: " + axiom); 797 addAxiom(ontology, axiom); 798 logger.info("Axiom added."); 799 logger.info("Running parameter Learner"); 800 edge.start(); 801 BigDecimal LL1 = edge.getLL(); 802 logger.info("Current Log-Likelihood: " + LL1); 803 if (LL1.compareTo(LL0) > 0) { 804 logger.info("Log-Likelihood enhanced. Updating ontologies..."); 805 OWLAnnotation annotation = df. 806 getOWLAnnotation(BundleUtilities.PROBABILISTIC_ANNOTATION_PROPERTY, 807 df.getOWLLiteral(edge.getParameter(axiom).doubleValue())); 808 OWLSubClassOfAxiom updatedAxiom = df.getOWLSubClassOfAxiom(axiom.getSubClass(), 809 axiom.getSuperClass(), Collections.singleton(annotation)); 810 learnedAxioms.add(updatedAxiom); 811 updateOntology(); 812 LL0 = LL1; 813 try { 814 MPIUtilities.sendBCastSignal(UPDATE, parameterLearnerComm); 815 logger.debug(myRank + " - Sent UPDATE signal to the slaves"); 816 } catch (MPIException e) { 817 String msg = myRank + " - Cannot send UPDATE signal to slaves: " + e.getMessage(); 818 logger.error(msg); 819 throw new StructureLearningException(msg); 820 } 821 } else { 822 logger.info("Log-Likelihood worsened. Removing Last Axiom..."); 823 removeAxiom(ontology, axiom); 824 try { 825 MPIUtilities.sendBCastSignal(REMOVE, parameterLearnerComm); 826 logger.debug(myRank + " - Sent REMOVE signal to the slaves"); 827 } catch (MPIException e) { 828 String msg = myRank + " - Cannot send REMOVE signal to slaves: " + e.getMessage(); 829 logger.error(msg); 830 throw new StructureLearningException(msg); 831 } 832 } 833 834 } catch (MPIException mpiEx) { 835 logger.error(myRank + " - Cannot perform greedy search: " + mpiEx.getMessage()); 836 throw new ParameterLearningException(mpiEx); 837 } catch (InconsistencyException iex) { 838 logger.info(iex.getMessage()); 839 logger.info("Trying with the next class expression"); 840 continue; 841 } 842 } 843 try { 844 MPIUtilities.sendBCastSignal(STOP, parameterLearnerComm); 845 logger.debug(myRank + " - Sent STOP signal to the slaves"); 846 } catch (MPIException mpiEx) { 847 logger.error(myRank + " - Cannot send stop signal: " + mpiEx.getMessage()); 848 throw new ParameterLearningException(mpiEx); 849 } 850 return new Revision(targetAxioms, revision.getBoolVars(), learnedAxioms, LL0); 851 } 852 853 private boolean terminationCriteriaSatisfied() { 854 boolean condition = stop 855 || currentIteration > maxIterations 856 || beamRevisions.isEmpty() 857 || currentDifferenceLL.compareTo(differenceLL) <= 0 858 || currentRatioLL.compareTo(ratioLL) <= 0; 859 860 if (stop) { 861 logger.info("Termination due to: STOP"); 862 } else if (currentIteration > maxIterations) { 863 logger.info("Termination due to: max iterations reached"); 864 } else if (beamRevisions.isEmpty()) { 865 logger.info("Termination due to: beam of revisions is empty"); 866 } else if (currentDifferenceLL.compareTo(differenceLL) <= 0) { 867 logger.info("Termination due to: minimum diffLL threshold reached"); 868 } else if (currentRatioLL.compareTo(ratioLL) <= 0) { 869 logger.info("Termination due to: minimum ratioLL threshold reached"); 870 } 871 872 return condition; 873 } 874 875 private void addAxiom(OWLOntology ontology, OWLAxiom axiom) throws InconsistencyException, MPIException { 876 877 OWLOntologyManager manager = ontology.getOWLOntologyManager(); 878 manager.addAxiom(ontology, axiom); 879 PelletReasoner pelletReasoner = new PelletReasonerFactory().createNonBufferingReasoner(ontology); 880 if (!pelletReasoner.isConsistent()) { 881 String message = "The axiom will make the KB inconsistent.\n" 882 + "It will NOT be added"; 883 logger.warn(message); 884 manager.removeAxiom(ontology, axiom); 885 throw new InconsistencyException(message); 886 } else { 887 try { 888 MPIUtilities.sendBCastSignal(START, parameterLearnerComm); 889 logger.debug(myRank + " - Sent START signal to slaves"); 890 int sentBytes = MPIUtilities.sendBCastObject(axiom, parameterLearnerComm); 891 logger.debug(myRank + " - Sent to slaves OWLAxiom object (" + sentBytes + " bytes)"); 892 } catch (MPIException e) { 893 logger.error(myRank + " - Cannot send axiom to EDGE slaves"); 894 throw new StructureLearningException(e); 895 } 896 } 897 } 898 899 private OWLAxiom recvAddAxiom(OWLOntology ontology) { 900 OWLAxiom axiom; 901 try { 902 Object obj = MPIUtilities.recvBCastObject(MASTER, parameterLearnerComm); 903 axiom = (OWLAxiom) obj; 904 } catch (MPIException e) { 905 String msg = myRank + " - Cannot receive axiom to add: " + e.getMessage(); 906 logger.error(msg); 907 throw new StructureLearningException(msg); 908 } 909 ontology.getOWLOntologyManager().addAxiom(ontology, axiom); 910 return axiom; 911 } 912 913 private void removeAxiom(OWLOntology ontology, OWLAxiom axiom) { 914 OWLOntologyManager manager = ontology.getOWLOntologyManager(); 915 manager.removeAxiom(ontology, axiom); 916 } 917 918 /** 919 * @return the edge 920 */ 921 public AbstractEDGEDistributed getEdge() { 922 return edge; 923 } 924 925 /** 926 * @param edge the edge to set 927 */ 928 @Autowired 929 public void setEdge(AbstractEDGEDistributed edge) { 930 this.edge = edge; 931 } 932 933 private void updateOntology() { 934 logger.debug("Updating ontology"); 935 OWLOntology ontology = edge.getLearnedOntology(); 936 edge.changeSourcesOntology(ontology); 937 logger.debug("Ontology Updated"); 938 } 939 940 private OWLOntology generateOntologyFromRevision(OWLOntology ontology, Revision revision) { 941 try { 942 OWLOntology revisionedOntology = BundleUtilities.copyOntology(ontology); 943 Set<OWLSubClassOfAxiom> learnedAxioms = revision.getLearnedAxioms(); 944 OWLOntologyManager manager = revisionedOntology.getOWLOntologyManager(); 945 manager.addAxioms(revisionedOntology, revision.getLearnedAxioms()); 946 int i = 0; 947 for (OWLAxiom axiom : revision.getTargetAxioms()) { 948 if (!revision.getBoolVars().get(i)) { 949 manager.removeAxiom(revisionedOntology, axiom); 950 } 951 } 952 return revisionedOntology; 953 } catch (OWLOntologyCreationException e) { 954 logger.error("Cannot refine ontology"); 955 throw new StructureLearningException(e); 956 } 957 } 958 959 private void printTimings(long totalTimeMills, long celaTimeMills, Map<String, Long> timeMap) { 960 logger.info("Main: " + totalTimeMills + " ms"); 961 logger.info("CELOE: " + celaTimeMills + " ms"); 962 logger.info("EDGE: " + (timeMap.get("EM") + timeMap.get("Bundle")) + " ms"); 963 logger.info("\tBundle: " + timeMap.get("Bundle") + " ms"); 964 logger.info("\tEM: " + timeMap.get("EM") + " ms"); 965 long timeOther = totalTimeMills - celaTimeMills - (timeMap.get("EM") + timeMap.get("Bundle")); 966 logger.info("Other: " + timeOther + " ms"); 967 logger.info("Program client: execution successfully terminated"); 968 } 969 970 private void updateTerminationCriteria() { 971 currentIteration++; 972 currentDifferenceLL = bestRevision.getLL().subtract(previousBestRevision.getLL()); 973 currentRatioLL = currentDifferenceLL.divide(previousBestRevision.getLL(), accuracy, BigDecimal.ROUND_HALF_UP); 974 } 975 976 private OWLOntology replaceSuperClass(OWLOntology finalOntology, Set<OWLSubClassOfAxiom> learnedAxioms) { 977 logger.debug(myRank + " - Replacing super class \"dummyClass\" with \"classToDescribe\""); 978 ClassLearningProblem clp = (ClassLearningProblem) cela.getLearningProblem(); 979 //Set<OWLSubClassOfAxiom> learnedAxioms = bestRevision.getLearnedAxioms(); 980 OWLOntologyManager man = finalOntology.getOWLOntologyManager(); 981 OWLDataFactory df = man.getOWLDataFactory(); 982 int numInitialAxioms = finalOntology.getAxiomCount(); 983 // remove the learned Axioms 984 //man.removeAxiom(finalOntology, learnedAxioms.iterator().next()); 985 Set<OWLSubClassOfAxiom> learnedAxiomsCopy = new LinkedHashSet<>(learnedAxioms); 986 for (OWLAxiom axiom : finalOntology.getAxioms(AxiomType.SUBCLASS_OF)) { 987 for (OWLAxiom axiomToRemove : learnedAxiomsCopy) { 988 // conviene usare una copia di probAddedAxioms 989 //in maniera tale da eliminare gli assiomi gi� trovati durante la ricerca e 990 //quindi ridurre il numero di check 991 //logger.debug("Learned axiom to remove: " + BundleUtilities.getManchesterSyntaxString(axiomToRemove)); 992 if (axiomToRemove.equalsIgnoreAnnotations(axiom)) { 993 man.removeAxiom(finalOntology, axiom); 994 learnedAxiomsCopy.remove(axiomToRemove); 995 break; 996 } 997 } 998 } 999 int numAxiomsAfterRemove = finalOntology.getAxiomCount(); 1000 // check if correctly removed 1001 if (numAxiomsAfterRemove != numInitialAxioms - learnedAxioms.size()) { 1002 String msg = myRank + " - Error during the replacement of super class: " 1003 + "Axiom remotion was incorrect. " 1004 + "numAxiomsAfterRemove: " + numAxiomsAfterRemove 1005 + " numInitialAxioms: " + numInitialAxioms 1006 + " numAxioms to remove: " + learnedAxioms.size() 1007 + " numAxioms removed: " + (numInitialAxioms - numAxiomsAfterRemove); 1008 logger.error(msg); 1009 throw new StructureLearningException(msg); 1010 } 1011 LinkedHashSet<OWLSubClassOfAxiom> newAxioms = new LinkedHashSet<>(); 1012 for (OWLSubClassOfAxiom axiom : learnedAxioms) { 1013 OWLSubClassOfAxiom newAxiom = df.getOWLSubClassOfAxiom(axiom.getSubClass(), 1014 clp.getClassToDescribe(), axiom.getAnnotations()); 1015 newAxioms.add(newAxiom); 1016 logger.info(myRank + " - Learned Axiom: " + newAxiom); 1017 } 1018 man.addAxioms(finalOntology, newAxioms); 1019 // check if correctly added 1020 if (finalOntology.getAxiomCount() != numAxiomsAfterRemove + learnedAxioms.size()) { 1021 String msg = myRank + " - Error during the replacement of super class: " 1022 + "Axiom addition was incorrect." 1023 + " numAxiomsAfterRemove: " + numAxiomsAfterRemove 1024 + " numAxioms to add: " + learnedAxioms.size() 1025 + " numAxioms added: " + (finalOntology.getAxiomCount() - numAxiomsAfterRemove);; 1026 logger.error(msg); 1027 throw new StructureLearningException(msg); 1028 } 1029 logger.debug(myRank + " - Replaced all the super classes"); 1030 return finalOntology; 1031 } 1032 1033 public String getName() { 1034 return "LEAPDistributed"; 1035 } 1036}