001/**
002 * Copyright (C) 2007-2011, Jens Lehmann
003 *
004 * This file is part of DL-Learner.
005 *
006 * DL-Learner is free software; you can redistribute it and/or modify
007 * it under the terms of the GNU General Public License as published by
008 * the Free Software Foundation; either version 3 of the License, or
009 * (at your option) any later version.
010 *
011 * DL-Learner is distributed in the hope that it will be useful,
012 * but WITHOUT ANY WARRANTY; without even the implied warranty of
013 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
014 * GNU General Public License for more details.
015 *
016 * You should have received a copy of the GNU General Public License
017 * along with this program.  If not, see <http://www.gnu.org/licenses/>.
018 *
019 */
020package org.dllearner.cli;
021
022import com.clarkparsia.owlapiv3.XSD;
023import com.google.common.base.StandardSystemProperty;
024import com.google.common.collect.Sets;
025import org.apache.jena.query.ResultSet;
026import org.apache.jena.rdf.model.*;
027import org.apache.jena.sparql.engine.http.QueryExceptionHTTP;
028import org.apache.jena.vocabulary.OWL;
029import org.apache.jena.vocabulary.RDF;
030import org.apache.jena.vocabulary.RDFS;
031import joptsimple.OptionException;
032import joptsimple.OptionParser;
033import joptsimple.OptionSet;
034import joptsimple.OptionSpec;
035import org.apache.jena.riot.checker.CheckerLiterals;
036import org.apache.jena.riot.system.ErrorHandlerFactory;
037import org.apache.log4j.Logger;
038import org.dllearner.algorithms.celoe.CELOE;
039import org.dllearner.algorithms.properties.AxiomAlgorithms;
040import org.dllearner.algorithms.properties.MultiPropertyAxiomLearner;
041import org.dllearner.configuration.spring.editors.ConfigHelper;
042import org.dllearner.core.*;
043import org.dllearner.kb.LocalModelBasedSparqlEndpointKS;
044import org.dllearner.kb.OWLAPIOntology;
045import org.dllearner.kb.SparqlEndpointKS;
046import org.dllearner.kb.sparql.*;
047import org.dllearner.accuracymethods.AccMethodFMeasure;
048import org.dllearner.learningproblems.AxiomScore;
049import org.dllearner.learningproblems.ClassLearningProblem;
050import org.dllearner.reasoning.ClosedWorldReasoner;
051import org.dllearner.reasoning.SPARQLReasoner;
052import org.dllearner.utilities.EnrichmentVocabulary;
053import org.dllearner.utilities.Helper;
054import org.dllearner.utilities.OwlApiJenaUtils;
055import org.dllearner.utilities.datastructures.SortedSetTuple;
056import org.dllearner.utilities.examples.AutomaticNegativeExampleFinderSPARQL2;
057import org.dllearner.utilities.owl.OWLAPIRenderers;
058import org.dllearner.utilities.owl.OWLEntityTypeAdder;
059import org.semanticweb.owlapi.apibinding.OWLManager;
060import org.semanticweb.owlapi.formats.ManchesterSyntaxDocumentFormat;
061import org.semanticweb.owlapi.formats.RDFXMLDocumentFormat;
062import org.semanticweb.owlapi.formats.TurtleDocumentFormat;
063import org.semanticweb.owlapi.io.SystemOutDocumentTarget;
064import org.semanticweb.owlapi.model.*;
065import org.semanticweb.owlapi.vocab.OWLRDFVocabulary;
066import uk.ac.manchester.cs.owl.owlapi.OWLDataFactoryImpl;
067
068import java.io.*;
069import java.lang.reflect.Field;
070import java.lang.reflect.InvocationTargetException;
071import java.math.BigInteger;
072import java.net.*;
073import java.security.SecureRandom;
074import java.text.DecimalFormat;
075import java.util.*;
076import java.util.Map.Entry;
077import java.util.concurrent.*;
078
079import static java.util.Arrays.asList;
080
081/**
082 * Command Line Interface for Enrichment.
083 *
084 * @author Jens Lehmann
085 *
086 */
087public class Enrichment {
088
089        // data structure for holding the result of an algorithm run
090        protected class AlgorithmRun {
091
092                // we only store the algorithm class and not the learning algorithm object,
093                // since otherwise we run into memory problems for full enrichment
094                private Class<? extends LearningAlgorithm> algorithm;
095                private List<EvaluatedAxiom<OWLAxiom>> axioms;
096                private Map<Field,Object> parameters;
097
098                public AlgorithmRun(Class<? extends LearningAlgorithm> algorithm, List<EvaluatedAxiom<OWLAxiom>> axioms, Map<Field,Object> parameters) {
099                        this.algorithm = algorithm;
100                        this.axioms = axioms;
101                        this.parameters = parameters;
102                }
103
104                public Class<? extends LearningAlgorithm> getAlgorithm() {
105                        return algorithm;
106                }
107
108                public List<EvaluatedAxiom<OWLAxiom>> getAxioms() {
109                        return axioms;
110                }
111
112                public Map<Field, Object> getParameters() {
113                        return parameters;
114                }
115        }
116
117        private static Logger logger = Logger.getLogger(Enrichment.class);
118        private DecimalFormat df = new DecimalFormat("##0.0");
119        private static final String DEFAULT_NS = "http://localhost:8080/";
120
121        //used to generate unique random identifiers
122        private SecureRandom random = new SecureRandom();
123
124        // enrichment parameters
125        private SparqlEndpointKS ks;
126        private OWLEntity resource;
127        private boolean verbose;
128
129        // max. execution time for each learner for each entity
130        private int maxExecutionTimeInSeconds = 10;
131
132        // restrict tested number of entities per type (only for testing purposes);
133        // should be set to -1 in production mode
134        int maxEntitiesPerType = -1;
135
136        // number of axioms which will be learned/considered (only applies to
137        // some learners)
138        private int nrOfAxiomsToLearn = 10;
139        private double threshold = 0.7;
140        private int chunksize = 1000;
141        private boolean omitExistingAxioms;
142        private List<String> allowedNamespaces = new ArrayList<>();
143        private int maxNrOfPositiveExamples = 20;
144        private int maxNrOfNegativeExamples = 20;
145
146        private boolean useInference;
147        private SPARQLReasoner reasoner;
148        private String cacheDir = "cache";
149
150        // lists of algorithms to apply
151        private List<Class<? extends LearningAlgorithm>> classAlgorithms;
152
153        // list of generated axioms while script is running
154        private List<AlgorithmRun> algorithmRuns;
155
156//      private CommonPrefixMap prefixes = new CommonPrefixMap();
157
158        // cache for SparqKnowledgeSource
159        KnowledgeSource ksCached;
160        AbstractReasonerComponent rcCached;
161
162        private Set<OWLAxiom> learnedOWLAxioms;
163        private Set<EvaluatedAxiom> learnedEvaluatedAxioms;
164        private boolean processPropertiesTypeInferred = false;
165        private boolean iterativeMode = false;
166
167        private boolean processObjectProperties;
168        private boolean processDataProperties;
169        private boolean processClasses;
170
171        AxiomLearningProgressMonitor progressMonitor = new ConsoleAxiomLearningProgressMonitor();
172
173        private OWLDataFactory dataFactory = new OWLDataFactoryImpl();
174
175        public Enrichment(SparqlEndpoint se, OWLEntity resource, double threshold, int nrOfAxiomsToLearn,
176                        boolean useInference, boolean verbose, int chunksize, int maxExecutionTimeInSeconds,
177                        boolean omitExistingAxioms) {
178                this(new SparqlEndpointKS(se), resource, threshold, nrOfAxiomsToLearn, useInference, verbose, chunksize,
179                                maxExecutionTimeInSeconds, omitExistingAxioms);
180        }
181
182        public Enrichment(SparqlEndpointKS ks, OWLEntity resource, double threshold, int nrOfAxiomsToLearn,
183                        boolean useInference, boolean verbose, int chunksize,
184                        int maxExecutionTimeInSeconds, boolean omitExistingAxioms) {
185                this.ks = ks;
186                this.resource = resource;
187                this.verbose = verbose;
188                this.threshold = threshold;
189                this.nrOfAxiomsToLearn = nrOfAxiomsToLearn;
190                this.useInference = useInference;
191                this.chunksize = chunksize;
192                this.maxExecutionTimeInSeconds = maxExecutionTimeInSeconds;
193                this.omitExistingAxioms = omitExistingAxioms;
194
195                try {
196                        ks.init();
197                } catch (ComponentInitException e1) {
198                        e1.printStackTrace();
199                }
200                
201                if(ks.isRemote()){
202                        try {
203                                cacheDir = "cache" + File.separator + URLEncoder.encode(ks.getEndpoint().getURL().toString(), "UTF-8");
204                        } catch (UnsupportedEncodingException e) {
205                                // TODO Auto-generated catch block
206                                e.printStackTrace();
207                        }
208                }
209
210                classAlgorithms = new LinkedList<>();
211//              classAlgorithms.add(DisjointClassesLearner.class);
212//              classAlgorithms.add(SimpleSubclassLearner.class);
213                classAlgorithms.add(CELOE.class);
214
215                algorithmRuns = new LinkedList<>();
216
217                learnedOWLAxioms = new HashSet<>();
218                learnedEvaluatedAxioms = new HashSet<>();
219        }
220
221        public void setAllowedNamespaces(List<String> allowedNamespaces) {
222                this.allowedNamespaces = allowedNamespaces;
223        }
224
225        /**
226         * @param iterativeMode the iterativeMode to set
227         */
228        public void setIterativeMode(boolean iterativeMode) {
229                this.iterativeMode = iterativeMode;
230        }
231
232        public EntityType<? extends OWLEntity> getEntityType(String resourceURI) {
233                EntityType<? extends OWLEntity> entityType = reasoner.getOWLEntityType(resourceURI);
234                if(entityType != null){
235                        return entityType;
236                } else {
237                        throw new IllegalArgumentException("Could not detect type of entity");
238                }
239        }
240
241        public void start() throws ComponentInitException, IllegalArgumentException, SecurityException {
242                reasoner = new SPARQLReasoner(ks);
243                reasoner.init();
244
245                if(useInference){
246                        System.out.print("Precomputing subsumption hierarchy ... ");
247                        long startTime = System.currentTimeMillis();
248                        reasoner.prepareSubsumptionHierarchy();
249                        System.out.println("done in " + (System.currentTimeMillis() - startTime) + " ms");
250                }
251
252                if(resource == null) {
253
254                        // loop over all entities and call appropriate algorithms
255                        Set<OWLProperty> processedProperties = new HashSet<>();
256                        if(processClasses){
257                                Set<OWLClass> classes = reasoner.getOWLClasses();
258                                filterByNamespaces(classes);
259                                processClasses(classes);
260                        }
261
262                        // process object properties
263                        if(processObjectProperties){
264                                Set<OWLObjectProperty> objectProperties = reasoner.getOWLObjectProperties();
265                                filterByNamespaces(objectProperties);
266                                processProperties(objectProperties, AxiomAlgorithms.getAxiomTypes(EntityType.OBJECT_PROPERTY));
267                                processedProperties.addAll(objectProperties);
268                        }
269
270                        // process data properties
271                        if(processDataProperties){
272                                Set<OWLDataProperty> dataProperties = reasoner.getOWLDataProperties();
273                                filterByNamespaces(dataProperties);
274                                processProperties(dataProperties, AxiomAlgorithms.getAxiomTypes(EntityType.DATA_PROPERTY));
275                                processedProperties.addAll(dataProperties);
276                        }
277
278                } else {
279                        System.out.println(resource + " appears to be a" + (resource.isOWLObjectProperty() ? "n " : " ")
280                                        + resource.getEntityType().getPrintName().toLowerCase()
281                                        + ". Running appropriate algorithms.\n");
282                        if(resource instanceof OWLObjectProperty) {
283                                processProperties(Collections.singleton(resource.asOWLObjectProperty()), AxiomAlgorithms.getAxiomTypes(EntityType.OBJECT_PROPERTY));
284                        } else if(resource instanceof OWLDataProperty) {
285                                processProperties(Collections.singleton(resource.asOWLDataProperty()), AxiomAlgorithms.getAxiomTypes(EntityType.DATA_PROPERTY));
286                        } else if(resource instanceof OWLClass) {
287                                processClasses(Collections.singleton(resource.asOWLClass()));
288                        } else {
289                                throw new Error("The type " + resource.getClass() + " of resource " + resource + " cannot be handled by this enrichment tool.");
290                        }
291                }
292        }
293
294        private void processClasses(Set<OWLClass> classes) {
295                for(OWLClass cls : classes) {
296                        try {
297                                runClassLearningAlgorithms(ks, cls);
298                        } catch (Exception e) {
299                                e.printStackTrace();
300                        }
301                }
302        }
303
304        private void processProperties(Set<? extends OWLProperty> properties, Set<AxiomType<? extends OWLAxiom>> axiomTypes){
305                MultiPropertyAxiomLearner la = new MultiPropertyAxiomLearner(ks);
306//              la.setUseSampling(true);
307                la.setProgressMonitor(progressMonitor);
308                la.setAxiomTypes(axiomTypes);
309                for(OWLProperty property : properties) {
310                        System.out.println("Processing property " + property.toStringID());
311                        la.setEntityToDescribe(property);
312                        la.start();
313
314                        for (AxiomType<? extends OWLAxiom> axiomType : axiomTypes) {
315
316                                List<EvaluatedAxiom<OWLAxiom>> evaluatedAxioms = la.getCurrentlyBestEvaluatedAxioms(axiomType, threshold);
317                                learnedEvaluatedAxioms.addAll(evaluatedAxioms);
318
319                                AbstractAxiomLearningAlgorithm algorithm = la.getAlgorithm(axiomType);
320                                
321                                if(algorithm != null) {
322                                        AlgorithmRun algorithmRun = new AlgorithmRun(
323                                                        AxiomAlgorithms.getAlgorithmClass(axiomType),
324                                                        evaluatedAxioms,
325                                                        ConfigHelper.getConfigOptionValues(la.getAlgorithm(axiomType)));
326                                        algorithmRuns.add(algorithmRun);
327                                } else {
328                                        // TODO what to do when algorithm failed
329                                }
330                                
331                        }
332                }
333        }
334
335        private <T extends OWLEntity> void filterByNamespaces(Collection<T> entities){
336                if(allowedNamespaces != null && !allowedNamespaces.isEmpty()){
337                        for (Iterator<T> iterator = entities.iterator(); iterator.hasNext();) {
338                                T entity = iterator.next();
339                                boolean startsWithAllowedNamespace = false;
340                                for (String ns : allowedNamespaces) {
341                                        if(entity.toStringID().startsWith(ns)){
342                                                startsWithAllowedNamespace = true;
343                                                break;
344                                        }
345                                }
346                                if(!startsWithAllowedNamespace){
347                                        iterator.remove();
348                                }
349                        }
350                }
351        }
352
353        private void filterByNamespaces(Model model){
354                List<Statement> toRemove = new ArrayList<>();
355                if(allowedNamespaces != null && !allowedNamespaces.isEmpty()){
356                        for (StmtIterator iterator = model.listStatements(); iterator.hasNext();) {
357                                Statement st = iterator.next();
358                                Property predicate = st.getPredicate();
359                                RDFNode object = st.getObject();
360                                boolean startsWithAllowedNamespace = false;
361                                if(predicate.equals(RDF.type) || predicate.equals(OWL.equivalentClass)){
362                                        if(object.isURIResource()){
363                                                for (String ns : allowedNamespaces) {
364                                                        if(object.asResource().getURI().startsWith(ns)){
365                                                                startsWithAllowedNamespace = true;
366                                                                break;
367                                                        }
368                                                }
369                                        } else {
370                                                startsWithAllowedNamespace = true;
371                                        }
372                                } else {
373                                        for (String ns : allowedNamespaces) {
374                                                if(predicate.getURI().startsWith(ns)){
375                                                        startsWithAllowedNamespace = true;
376                                                        break;
377                                                }
378                                        }
379                                }
380                                if(!startsWithAllowedNamespace){
381                                        toRemove.add(st);
382                                }
383                        }
384                }
385                model.remove(toRemove);
386        }
387
388        @SuppressWarnings("unchecked")
389        private void runClassLearningAlgorithms(SparqlEndpointKS ks, OWLClass nc) throws ComponentInitException {
390                System.out.println("Running algorithms for class " + nc);
391                for (Class<? extends LearningAlgorithm> algorithmClass : classAlgorithms) {
392                        if(algorithmClass == CELOE.class) {
393//                              applyCELOE(ks, nc, false, false);
394//                              applyCELOE(ks, nc, true, true);
395                                applyCELOE(ks, nc, true, false);
396                        } else {
397                                applyLearningAlgorithm((Class<AxiomLearningAlgorithm>)algorithmClass, ks, nc);
398                        }
399                }
400        }
401
402        private List<EvaluatedAxiom<OWLAxiom>> applyCELOE(SparqlEndpointKS ks, OWLClass nc, boolean equivalence, boolean reuseKnowledgeSource) throws ComponentInitException {
403                // get instances of class as positive examples
404                System.out.print("finding positives ... ");
405                long startTime = System.currentTimeMillis();
406                SortedSet<OWLIndividual> posExamples = reasoner.getIndividuals(nc, maxNrOfPositiveExamples);
407                long runTime = System.currentTimeMillis() - startTime;
408                if(posExamples.isEmpty()){
409                        System.out.println("Skipping CELOE because class " + nc.toString() + " is empty.");
410                        return Collections.emptyList();
411                }
412                SortedSet<String> posExStr = Helper.getStringSet(posExamples);
413                System.out.println("done (" + posExStr.size()+ " examples found in " + runTime + " ms)");
414
415                // use own implementation of negative example finder
416                System.out.print("finding negatives ... ");
417                startTime = System.currentTimeMillis();
418                AutomaticNegativeExampleFinderSPARQL2 finder = new AutomaticNegativeExampleFinderSPARQL2(reasoner);
419                SortedSet<OWLIndividual> negExamples = finder.getNegativeExamples(nc, posExamples, maxNrOfNegativeExamples);
420                SortedSetTuple<OWLIndividual> examples = new SortedSetTuple<>(posExamples, negExamples);
421                runTime = System.currentTimeMillis() - startTime;
422                System.out.println("done (" + negExamples.size()+ " examples found in " + runTime + " ms)");
423
424                AbstractReasonerComponent rc;
425                KnowledgeSource ksFragment;
426                if(reuseKnowledgeSource){
427                        ksFragment = ksCached;
428                        rc = rcCached;
429                } else {
430                        System.out.print("extracting fragment ... ");//org.apache.jena.shared.impl.JenaParameters.enableEagerLiteralValidation = true;
431                        startTime = System.currentTimeMillis();
432                        Model model;
433                        if(ks.isRemote()){
434//                              model = getFragmentMultithreaded(ks, Sets.union(posExamples, negExamples));
435                                model = getFragment(ks, Sets.union(posExamples, negExamples));
436                        } else {
437                                model = ((LocalModelBasedSparqlEndpointKS)ks).getModel();
438                        }
439
440                        filter(model);
441                        filterByNamespaces(model);
442                        OWLEntityTypeAdder.addEntityTypes(model);
443
444                        runTime = System.currentTimeMillis() - startTime;
445                        System.out.println("done (" + model.size()+ " triples found in " + runTime + " ms)");
446                        OWLOntology ontology = asOWLOntology(model);
447                        if(reasoner.getClassHierarchy() != null){
448                                ontology.getOWLOntologyManager().addAxioms(ontology, reasoner.getClassHierarchy().toOWLAxioms());
449                        }
450                        ksFragment = new OWLAPIOntology(ontology);
451                        try {
452                                OWLManager.createOWLOntologyManager().saveOntology(ontology, new TurtleDocumentFormat(), new FileOutputStream(System.getProperty("java.io.tmpdir") + File.separator + "test.ttl"));
453                        } catch (OWLOntologyStorageException | FileNotFoundException e) {
454                                e.printStackTrace();
455                        }
456//                      ksFragment.init();
457                        System.out.println("Init reasoner");
458                        rc = new ClosedWorldReasoner(ksFragment);
459                        rc.init();
460                        System.out.println("Finished init reasoner");
461//                      rc.setSubsumptionHierarchy(reasoner.getClassHierarchy());
462                        ksCached = ksFragment;
463                        rcCached = rc;
464//                      for (Individual ind : posExamples) {
465//                              System.out.println(ResultSetFormatter.asText(org.apache.jena.query.QueryExecutionFactory.create("SELECT * WHERE {<" + ind.getName() + "> ?p ?o. OPTIONAL{?o a ?o_type}}",model).execSelect()));
466//                      }
467                }
468
469        ClassLearningProblem lp = new ClassLearningProblem(rc);
470                lp.setClassToDescribe(nc);
471        lp.setEquivalence(equivalence);
472        lp.setAccuracyMethod(new AccMethodFMeasure(true));
473        lp.setMaxExecutionTimeInSeconds(10);
474        lp.init();
475
476        CELOE la = new CELOE(lp, rc);
477        la.setMaxExecutionTimeInSeconds(10);
478        la.setNoisePercentage(25);
479        la.setMaxNrOfResults(100);
480        la.init();
481//        ((RhoDRDown)la.getOperator()).setUseNegation(false);
482        startTime = System.currentTimeMillis();
483        System.out.print("running CELOE (for " + (equivalence ? "equivalent classes" : "sub classes") + ") ... ");
484        la.start();
485        runTime = System.currentTimeMillis() - startTime;
486        System.out.println("done in " + runTime + " ms");
487
488        // convert the result to axioms (to make it compatible with the other algorithms)
489        List<? extends EvaluatedDescription<? extends Score>> learnedDescriptions = la.getCurrentlyBestEvaluatedDescriptions(threshold);
490        List<EvaluatedAxiom<OWLAxiom>> learnedAxioms = new LinkedList<>();
491        for(EvaluatedDescription<? extends Score> learnedDescription : learnedDescriptions) {
492                OWLAxiom axiom;
493                if(equivalence) {
494                        axiom = dataFactory.getOWLEquivalentClassesAxiom(nc, learnedDescription.getDescription());
495                } else {
496                        axiom = dataFactory.getOWLSubClassOfAxiom(nc, learnedDescription.getDescription());
497                }
498                Score score = lp.computeScore(learnedDescription.getDescription());
499                learnedAxioms.add(new EvaluatedAxiom<>(axiom, new AxiomScore(score.getAccuracy())));
500        }
501        System.out.println(prettyPrint(learnedAxioms));
502        learnedEvaluatedAxioms.addAll(learnedAxioms);
503        algorithmRuns.add(new AlgorithmRun(CELOE.class, learnedAxioms, ConfigHelper.getConfigOptionValues(la)));
504                return learnedAxioms;
505        }
506
507        private Model getFragment(SparqlEndpointKS ks, Set<OWLIndividual> individuals){
508                ConciseBoundedDescriptionGenerator cbdGen = new ConciseBoundedDescriptionGeneratorImpl(ks.getQueryExecutionFactory());
509                Model model = ModelFactory.createDefaultModel();
510                for(OWLIndividual ind : individuals){
511                        Model cbd = cbdGen.getConciseBoundedDescription(ind.toStringID(), 2);
512                        model.add(cbd);
513                }
514                return model;
515        }
516
517        private Model getFragmentMultithreaded(final SparqlEndpointKS ks, Set<OWLIndividual> individuals){
518                Model model = ModelFactory.createDefaultModel();
519                ExecutorService threadPool = Executors.newFixedThreadPool(Runtime.getRuntime().availableProcessors());
520                List<Future<Model>> futures = new ArrayList<>();
521                for (final OWLIndividual ind : individuals) {
522                        futures.add(threadPool.submit(new Callable<Model>() {
523                                @Override
524                                public Model call() throws Exception {
525                                        ConciseBoundedDescriptionGenerator cbdGen = new ConciseBoundedDescriptionGeneratorImpl(ks.getQueryExecutionFactory());
526                                        return cbdGen.getConciseBoundedDescription(ind.toStringID(), 2);
527                                }
528                        }));
529                }
530                for (Future<Model> future : futures) {
531                        try {
532                                model.add(future.get());
533                        } catch (InterruptedException | ExecutionException e) {
534                                e.printStackTrace();
535                        }
536                }
537                threadPool.shutdown();
538                return model;
539        }
540
541        private List<EvaluatedAxiom<OWLAxiom>> applyLearningAlgorithm(Class<? extends AxiomLearningAlgorithm> algorithmClass, SparqlEndpointKS ks, OWLEntity entity) throws ComponentInitException {
542                AxiomLearningAlgorithm learner = null;
543                try {
544                        learner = algorithmClass.getConstructor(
545                                        SparqlEndpointKS.class).newInstance(ks);
546                } catch (Exception e) {
547                        e.printStackTrace();
548                }
549                if(classAlgorithms.contains(algorithmClass)) {
550
551                        ConfigHelper.configure(learner, "classToDescribe", entity);
552                } else {
553                        ConfigHelper.configure(learner, "propertyToDescribe", entity);
554                }
555                ConfigHelper.configure(learner, "maxExecutionTimeInSeconds",
556                                maxExecutionTimeInSeconds);
557                ((AbstractAxiomLearningAlgorithm)learner).setReturnOnlyNewAxioms(omitExistingAxioms);
558                learner.init();
559                if(reasoner != null){
560                        ((AbstractAxiomLearningAlgorithm)learner).setReasoner(reasoner);
561                }
562                String algName = AnnComponentManager.getName(learner);
563                System.out.print("Applying " + algName + " on " + entity + " ... ");
564                long startTime = System.currentTimeMillis();
565                try {
566                        learner.start();
567                } catch (Exception e) {
568                        if(e.getCause() instanceof SocketTimeoutException){
569                                System.out.println("Query timed out (endpoint possibly too slow).");
570                        } else {
571                                e.printStackTrace();
572                        }
573                }
574                long runtime = System.currentTimeMillis() - startTime;
575                System.out.println("done in " + runtime + " ms");
576                List<EvaluatedAxiom<OWLAxiom>> learnedAxioms = learner
577                                .getCurrentlyBestEvaluatedAxioms(nrOfAxiomsToLearn, threshold);
578                System.out.println(prettyPrint(learnedAxioms));
579                learnedEvaluatedAxioms.addAll(learnedAxioms);
580                for(EvaluatedAxiom<OWLAxiom> evAx : learnedAxioms){
581                        learnedOWLAxioms.add(evAx.getAxiom());
582                }
583
584                algorithmRuns.add(new AlgorithmRun(learner.getClass(), learnedAxioms, ConfigHelper.getConfigOptionValues(learner)));
585                return learnedAxioms;
586        }
587
588        private String prettyPrint(List<EvaluatedAxiom<OWLAxiom>> learnedAxioms) {
589                String str = "suggested axioms and their score in percent:\n";
590                if(learnedAxioms.isEmpty()) {
591                        return "  no axiom suggested\n";
592                } else {
593                        for (EvaluatedAxiom<OWLAxiom> learnedAxiom : learnedAxioms) {
594                                str += " " + prettyPrint(learnedAxiom) + "\n";
595                        }
596                }
597                return str;
598        }
599
600        private String prettyPrint(EvaluatedAxiom<OWLAxiom> axiom) {
601                double acc = axiom.getScore().getAccuracy() * 100;
602                String accs = df.format(acc);
603                if(accs.length()==3) { accs = "  " + accs; }
604                if(accs.length()==4) { accs = " " + accs; }
605                String str =  accs + "%\t" + OWLAPIRenderers.toManchesterOWLSyntax(axiom.getAxiom());
606                return str;
607        }
608
609        /*
610         * Generates list of OWL axioms.
611         */
612        List<OWLAxiom> toRDF(List<EvaluatedAxiom<OWLAxiom>> evalAxioms, Class<? extends LearningAlgorithm> algorithm, Map<Field, Object> parameters, SparqlEndpointKS ks){
613                return toRDF(evalAxioms, algorithm, parameters, ks, null);
614        }
615
616        private List<OWLAxiom> toRDF(List<EvaluatedAxiom<OWLAxiom>> evalAxioms, Class<? extends LearningAlgorithm> algorithm, Map<Field,Object> parameters, SparqlEndpointKS ks, String defaultNamespace){
617                if(defaultNamespace == null || defaultNamespace.isEmpty()){
618                        defaultNamespace = DEFAULT_NS;
619                }
620                List<OWLAxiom> axioms = new ArrayList<>();
621
622                OWLDataFactory f = new OWLDataFactoryImpl();
623
624                //create instance for suggestion set
625                String suggestionSetID = defaultNamespace + generateId();
626                OWLIndividual ind = f.getOWLNamedIndividual(IRI.create(suggestionSetID));
627                //add type SuggestionSet
628                OWLAxiom ax = f.getOWLClassAssertionAxiom(EnrichmentVocabulary.SuggestionSet, ind);
629                axioms.add(ax);
630
631                //create instance for algorithm run
632                String algorithmRunID = defaultNamespace + generateId();
633                OWLIndividual algorithmRunInd = f.getOWLNamedIndividual(IRI.create(algorithmRunID));
634                //add type AlgorithmRun
635                ax = f.getOWLClassAssertionAxiom(EnrichmentVocabulary.AlgorithmRun, algorithmRunInd);
636                axioms.add(ax);
637                //generate instance for algorithm
638                String algorithmName = AnnComponentManager.getName(algorithm);
639                String algorithmID = "http://dl-learner.org#" + algorithmName.replace(" ", "_");
640                OWLIndividual algorithmInd = f.getOWLNamedIndividual(IRI.create(algorithmID));
641                //add label to algorithm instance
642                OWLAnnotation labelAnno = f.getOWLAnnotation(
643                                f.getOWLAnnotationProperty(OWLRDFVocabulary.RDFS_LABEL.getIRI()),
644                                f.getOWLLiteral(algorithmName));
645                ax = f.getOWLAnnotationAssertionAxiom(algorithmInd.asOWLNamedIndividual().getIRI(), labelAnno);
646                axioms.add(ax);
647                //add version to algorithm
648                ax = f.getOWLDataPropertyAssertionAxiom(EnrichmentVocabulary.version, algorithmInd, algorithm.getAnnotation(ComponentAnn.class).version());
649                axioms.add(ax);
650                //add algorithm instance to algorithm run instance
651                ax = f.getOWLObjectPropertyAssertionAxiom(EnrichmentVocabulary.usedAlgorithm,
652                                algorithmRunInd, algorithmInd);
653                axioms.add(ax);
654                //add Parameters to algorithm run instance
655                OWLIndividual paramInd;
656                for(Entry<Field, Object> entry : parameters.entrySet()){
657                        paramInd = f.getOWLNamedIndividual(IRI.create(generateId()));
658                        ax = f.getOWLClassAssertionAxiom(EnrichmentVocabulary.Parameter, paramInd);
659                        axioms.add(ax);
660                        ax = f.getOWLDataPropertyAssertionAxiom(EnrichmentVocabulary.parameterName, paramInd, AnnComponentManager.getName(entry.getKey()));
661                        axioms.add(ax);
662                        ax = f.getOWLDataPropertyAssertionAxiom(EnrichmentVocabulary.parameterValue, paramInd, entry.getValue().toString());
663                        axioms.add(ax);
664                }
665                //add timestamp
666                ax = f.getOWLDataPropertyAssertionAxiom(EnrichmentVocabulary.timestamp, algorithmRunInd, System.currentTimeMillis());
667                axioms.add(ax);
668
669                //add used input to algorithm run instance
670                OWLNamedIndividual knowldegeBaseInd = f.getOWLNamedIndividual(IRI.create(ks.getEndpoint().getURL()));
671                ax = f.getOWLClassAssertionAxiom(EnrichmentVocabulary.SPARQLEndpoint, knowldegeBaseInd);
672                axioms.add(ax);
673                if(!ks.getEndpoint().getDefaultGraphURIs().isEmpty()) {
674                        // TODO: only writes one default graph
675                        ax = f.getOWLObjectPropertyAssertionAxiom(EnrichmentVocabulary.defaultGraph, knowldegeBaseInd, f.getOWLNamedIndividual(IRI.create(ks.getEndpoint().getDefaultGraphURIs().iterator().next())));
676                        axioms.add(ax);
677                }
678                ax = f.getOWLObjectPropertyAssertionAxiom(EnrichmentVocabulary.hasInput,
679                                algorithmRunInd, knowldegeBaseInd);
680                axioms.add(ax);
681
682                //add algorithm run instance to suggestion set instance via ObjectProperty creator
683                ax = f.getOWLObjectPropertyAssertionAxiom(EnrichmentVocabulary.creator,
684                                ind, algorithmRunInd);
685                axioms.add(ax);
686
687                //add suggestions to suggestions set
688                Entry<OWLIndividual, List<OWLAxiom>> ind2Axioms;
689                for(EvaluatedAxiom evAx : evalAxioms){
690                        Map<OWLIndividual, List<OWLAxiom>> map = evAx.toRDF(defaultNamespace);
691                        ind2Axioms = map.entrySet().iterator().next();
692                        ax = f.getOWLObjectPropertyAssertionAxiom(EnrichmentVocabulary.hasSuggestion, ind, ind2Axioms.getKey());
693                        axioms.add(ax);
694                        axioms.addAll(ind2Axioms.getValue());
695                }
696
697//              printManchesterOWLSyntax(axioms, defaultNamespace);
698//              printTurtleSyntax(axioms);
699//              printNTriplesSyntax(axioms);
700                return axioms;
701        }
702
703          private String generateId(){
704            return new BigInteger(130, random).toString(32);
705          }
706
707        /*
708         * Write axioms in Manchester OWL Syntax.
709         */
710        private void printManchesterOWLSyntax(List<OWLAxiom> axioms, String defaultNamespace){
711                try {
712                        System.out.println("ENRICHMENT[");
713
714                        ManchesterSyntaxDocumentFormat manSyntaxFormat = new ManchesterSyntaxDocumentFormat();
715                        manSyntaxFormat.setDefaultPrefix(defaultNamespace);
716                        manSyntaxFormat.setPrefix("enrichment", "http://www.dl-learner.org/enrichment.owl#");
717
718                        OWLOntologyManager man = OWLManager.createOWLOntologyManager();
719                        OWLOntology ontology = man.createOntology(new HashSet<>(axioms), IRI.create(defaultNamespace + "enrichment"));
720                        OWLManager.createOWLOntologyManager().saveOntology(ontology, manSyntaxFormat, new SystemOutDocumentTarget());
721
722                        System.out.println("]");
723                } catch (OWLOntologyCreationException | OWLOntologyStorageException e) {
724                        e.printStackTrace();
725                }
726        }
727
728//      private Model getModel(List<OWLAxiom> axioms) {
729//              Model model = ModelFactory.createDefaultModel();
730//              try {
731//                      Conversion.OWLAPIOntology2JenaModel(OWLManager.createOWLOntologyManager().createOntology(new HashSet<OWLAxiom>(axioms)), model);
732//              } catch (OWLOntologyCreationException e) {
733//                      e.printStackTrace();
734//              }
735//              return model;
736//      }
737
738        private OWLOntology asOWLOntology(Model model) {
739                try {
740                        FileOutputStream fos = null;
741                        try {
742                                fos = new FileOutputStream("bug.ttl");
743                        } catch (FileNotFoundException e) {
744                                e.printStackTrace();
745                        }
746                        ByteArrayOutputStream baos = new ByteArrayOutputStream();
747                        model.write(baos, "TURTLE", null);
748                        model.write(fos, "TURTLE", null);
749                        OWLOntologyManager man = OWLManager.createOWLOntologyManager();
750                        OWLOntology ontology = man.loadOntologyFromOntologyDocument(new ByteArrayInputStream(baos.toByteArray()));
751                        return ontology;
752                } catch (OWLOntologyCreationException e) {
753                        e.printStackTrace();
754                        try {
755                                model.write(new FileOutputStream("parse-error.ttl"), "TURTLE", null);
756                        } catch (FileNotFoundException e1) {
757                                e1.printStackTrace();
758                        }
759                }
760                return null;
761        }
762
763        private void filter(Model model) {
764                // filter out triples with String literals, as therein often occur
765                // some syntax errors and they are not relevant for learning
766                List<Statement> statementsToRemove = new ArrayList<>();
767                List<Statement> statementsToAdd = new ArrayList<>();
768                for (Statement st : model.listStatements().toList()) {
769                        RDFNode subject = st.getSubject();
770                        RDFNode object = st.getObject();
771
772                        if (object.isAnon()) {
773                                if (!model.listStatements(object.asResource(), null, (RDFNode) null).hasNext()) {
774                                        statementsToRemove.add(st);
775                                }
776                        } else if (st.getPredicate().equals(RDF.type) &&
777                                        (object.equals(RDFS.Class.asNode()) || object.equals(OWL.Class.asNode()) || object.equals(RDFS.Literal.asNode()))) {
778                                //remove statements like <x a owl:Class>
779                                statementsToRemove.add(st);
780                        } else {
781                                // fix URIs with spaces
782                                Resource newSubject = (Resource) subject;
783                                RDFNode newObject = object;
784                                boolean validTriple = true;
785                                if (subject.isURIResource()) {
786                                        String uri = subject.asResource().getURI();
787                                        if (uri.contains(" ")) {
788                                                newSubject = model.createResource(uri.replace(" ", ""));
789                                        }
790                                }
791                                if (object.isURIResource()) {
792                                        String uri = object.asResource().getURI();
793                                        if (uri.contains(" ")) {
794                                                newObject = model.createResource(uri.replace(" ", ""));
795                                        }
796                                }
797                                if (object.isLiteral()) {
798                                        Literal lit = object.asLiteral();
799                                        if (lit.getDatatype() == null || lit.getDatatype().equals(XSD.STRING)) {
800                                                newObject = model.createLiteral("shortened", "en");
801                                        }
802                                        validTriple = CheckerLiterals.checkLiteral(object.asNode(), ErrorHandlerFactory.errorHandlerNoLogging, 1L, 1L);
803                                }
804                                if (validTriple) {
805                                        statementsToAdd.add(model.createStatement(newSubject, st.getPredicate(), newObject));
806                                }
807                                statementsToRemove.add(st);
808                        }
809
810                }
811                model.remove(statementsToRemove);
812                model.add(statementsToAdd);
813        }
814
815        Model getModel(List<OWLAxiom> axioms) {
816                try {
817                        OWLOntology ontology = OWLManager.createOWLOntologyManager().createOntology(new HashSet<>(axioms));
818                        Model model = OwlApiJenaUtils.getModel(ontology);
819                        model.setNsPrefix("enr", "http://www.dl-learner.org/enrichment.owl#");
820                        return model;
821                } catch (OWLOntologyCreationException e) {
822                        e.printStackTrace();
823                }
824                return null;
825        }
826
827        public OWLOntology getGeneratedOntology(){
828                OWLOntology ontology = null;
829                try {
830                        OWLOntologyManager man = OWLManager.createOWLOntologyManager();
831                        ontology = man.createOntology(learnedOWLAxioms);
832                } catch (OWLOntologyCreationException e) {
833                        // TODO Auto-generated catch block
834                        e.printStackTrace();
835                }
836                return ontology;
837        }
838
839        public OWLOntology getGeneratedOntology(boolean withConfidenceAsAnnotations){
840                OWLOntology ontology = null;
841                try {
842                        OWLOntologyManager man = OWLManager.createOWLOntologyManager();
843                        OWLDataFactory factory = man.getOWLDataFactory();
844
845                        OWLAnnotationProperty confAnnoProp = factory.getOWLAnnotationProperty(IRI.create(EnrichmentVocabulary.NS
846                                        + "confidence"));
847                        Set<OWLAxiom> axioms = new HashSet<>();
848                        for (EvaluatedAxiom evAx : learnedEvaluatedAxioms) {
849                                OWLAxiom ax = evAx.getAxiom();
850                                if (withConfidenceAsAnnotations) {
851                                        ax = ax.getAnnotatedAxiom(Collections.singleton(factory.getOWLAnnotation(confAnnoProp,
852                                                        factory.getOWLLiteral(evAx.getScore().getAccuracy()))));
853                                }
854                                axioms.add(ax);
855                        }
856                        ontology = man.createOntology(axioms);
857
858                } catch (OWLOntologyCreationException e) {
859                        e.printStackTrace();
860                }
861                return ontology;
862        }
863
864        /*
865         * Write axioms in Turtle syntax.
866         */
867        private void printTurtleSyntax(List<OWLAxiom> axioms){
868                try {
869                        System.out.println("ENRICHMENT[");
870                        Model model = OwlApiJenaUtils.getModel(OWLManager.createOWLOntologyManager().createOntology(new HashSet<>(axioms)));
871                        model.write(System.out, "TURTLE");
872                        System.out.println("]");
873                } catch (OWLOntologyCreationException e) {
874                        // TODO Auto-generated catch block
875                        e.printStackTrace();
876                }
877        }
878
879        /*
880         * Write axioms in Turtle syntax.
881         */
882        private void printNTriplesSyntax(List<OWLAxiom> axioms){
883                try {
884                        System.out.println("ENRICHMENT[");
885                        Model model = OwlApiJenaUtils.getModel(OWLManager.createOWLOntologyManager().createOntology(new HashSet<>(axioms)));
886                        model.write(System.out, "N-TRIPLES");
887                        System.out.println("]");
888                } catch (OWLOntologyCreationException e) {
889                        // TODO Auto-generated catch block
890                        e.printStackTrace();
891                }
892        }
893
894        public List<AlgorithmRun> getAlgorithmRuns() {
895                return algorithmRuns;
896        }
897
898        /**
899         * @param processClasses the processClasses to set
900         */
901        public void setProcessClasses(boolean processClasses) {
902                this.processClasses = processClasses;
903        }
904
905        /**
906         * @param processDataProperties the processDataProperties to set
907         */
908        public void setProcessDataProperties(boolean processDataProperties) {
909                this.processDataProperties = processDataProperties;
910        }
911
912        /**
913         * @param processObjectProperties the processObjectProperties to set
914         */
915        public void setProcessObjectProperties(boolean processObjectProperties) {
916                this.processObjectProperties = processObjectProperties;
917        }
918
919        /**
920         * @param processPropertiesTypeInferred the processPropertiesTypeInferred to set
921         */
922        public void setProcessPropertiesTypeInferred(boolean processPropertiesTypeInferred) {
923                this.processPropertiesTypeInferred = processPropertiesTypeInferred;
924        }
925
926        public static void main(String[] args) throws IOException, ComponentInitException, IllegalArgumentException, SecurityException, InstantiationException, IllegalAccessException, InvocationTargetException, NoSuchMethodException, LearningProblemUnsupportedException {
927                OptionParser parser = new OptionParser();
928                parser.acceptsAll(asList("h", "?", "help"), "Show help.");
929//              parser.acceptsAll(asList("v", "verbose"), "Verbosity level.").withOptionalArg().ofType(Boolean.class).defaultsTo(false);
930                parser.acceptsAll(asList("e", "endpoint"), "SPARQL endpoint URL to be used.")
931                                .withRequiredArg().ofType(URL.class);
932                parser.acceptsAll(asList("g", "graph"),
933                                "URI of default graph for queries on SPARQL endpoint.").withOptionalArg()
934                                .ofType(URI.class);
935                parser.acceptsAll(asList("r", "resource"),
936                                "The resource for which enrichment axioms should be suggested.").withOptionalArg().ofType(URI.class);
937                parser.acceptsAll(asList("o", "output"), "Specify a file where the output can be written.")
938                                .withOptionalArg().ofType(File.class);
939                // TODO: other interesting formats: html, manchester, sparul
940                parser.acceptsAll(asList("f", "format"),
941                                "Format of the generated output (plain, rdf/xml, turtle, n-triples).").withOptionalArg()
942                                .ofType(String.class).defaultsTo("plain");
943                parser.acceptsAll(asList("t", "threshold"),
944                                "Confidence threshold for suggestions. Set it to a value between 0 and 1.").withOptionalArg()
945                                .ofType(Double.class).defaultsTo(0.7);
946                parser.acceptsAll(asList("l", "limit"),
947                "Maximum number of returned axioms per axiom type. Set it to -1 if all axioms above the threshold should be returned.").withOptionalArg()
948                .ofType(Integer.class).defaultsTo(10);
949                parser.acceptsAll(asList("i", "inference"),
950                                "Specifies whether to use inference. If yes, the schema will be loaded into a reasoner and used for computing the scores.").withOptionalArg().ofType(Boolean.class).defaultsTo(false);
951                parser.acceptsAll(asList("iterative"),
952                                "Specifies whether to use local fragments or single query mode.").withOptionalArg().ofType(Boolean.class).defaultsTo(false);
953                parser.acceptsAll(asList("s", "serialize"), "Specify a file where the ontology with all axioms can be written.")
954                .withRequiredArg().ofType(File.class);
955                parser.acceptsAll(asList("a", "annotations"),
956                                "Specifies whether to save scores as annotations.").withOptionalArg().ofType(Boolean.class).defaultsTo(true);
957                parser.acceptsAll(asList("chunksize"),
958                "Specifies the chunk size for the query result as the approach is incrementally.").withRequiredArg().ofType(Integer.class).defaultsTo(1000);
959                parser.acceptsAll(asList("maxExecutionTimeInSeconds"),
960                "Specifies the max execution time for each algorithm run and each entity.").withRequiredArg().ofType(Integer.class).defaultsTo(10);
961                parser.acceptsAll(asList("omitExistingAxioms"),
962                                "Specifies whether return only axioms which not already exist in the knowlegde base.").withOptionalArg().ofType(Boolean.class).defaultsTo(false);
963                OptionSpec<String> allowedNamespacesOption = parser.accepts( "ns" ).withRequiredArg().ofType( String.class )
964                    .withValuesSeparatedBy( ',' );
965
966                parser.acceptsAll(asList("op"),
967                                "Specifies whether to compute axiom for object properties.").withOptionalArg().ofType(Boolean.class).defaultsTo(true);
968                parser.acceptsAll(asList("dp"),
969                                "Specifies whether to compute axiom for data properties.").withOptionalArg().ofType(Boolean.class).defaultsTo(true);
970                parser.acceptsAll(asList("cls"),
971                                "Specifies whether compute axiom for classes.").withOptionalArg().ofType(Boolean.class).defaultsTo(true);
972
973                //username and password if endpoint is protected
974                parser.acceptsAll(asList("u", "username"), "Specify the username.")
975                .withOptionalArg().ofType(String.class);
976                parser.acceptsAll(asList("pw", "password"), "Specify the password.")
977                .withOptionalArg().ofType(String.class);
978
979                // parse options and display a message for the user in case of problems
980                OptionSet options = null;
981
982                if (args.length == 0) {
983                    parser.printHelpOn(System.out);
984                    System.exit(0);
985                }
986
987                try {
988                        options = parser.parse(args);
989                } catch (Exception e) {
990                        System.out.println("Error: " + e.getMessage() + ". Use -? to get help.");
991                        System.exit(0);
992                }
993
994                // print help screen
995                if (options.has("?")) {
996                        parser.printHelpOn(System.out);
997                        String addHelp = "Additional explanations: The resource specified should " +
998                        "be a class, object \nproperty or data property. DL-Learner will try to " +
999                        "automatically detect its \ntype. If no resource is specified, DL-Learner will " +
1000                        "generate enrichment \nsuggestions for all detected classes and properties in " +
1001                        "the given endpoint \nand graph. This can take several hours.";
1002                        System.out.println();
1003                        System.out.println(addHelp);
1004                        // main script
1005                } else {
1006                        // check that endpoint was specified
1007                        if(!options.hasArgument("endpoint")) {
1008                                System.out.println("Please specify a SPARQL endpoint (using the -e option).");
1009                                System.exit(0);
1010                        }
1011
1012                        SparqlEndpointKS ks = null;
1013                        // create SPARQL endpoint object (check that indeed a URL was given)
1014                        URL endpoint = null;
1015                        try {
1016                                endpoint = (URL) options.valueOf("endpoint");
1017                        } catch(OptionException e) {
1018                                System.out.println("The specified endpoint appears not be a proper URL.");
1019                                System.exit(0);
1020                        }
1021                        //check if the URL is a file and if exists load it into a JENA model
1022                        try {
1023                                if(isLocalFile(endpoint)){
1024                                        File file = new File(endpoint.toURI());
1025                                        if(file.exists()){
1026                                                Model kbModel = ModelFactory.createDefaultModel();
1027                                                kbModel.read(new FileInputStream(file), null);
1028                                                ks = new LocalModelBasedSparqlEndpointKS(kbModel);
1029                                        }
1030                                } else {
1031                                        URI graph = null;
1032                                        try {
1033                                                graph = (URI) options.valueOf("graph");
1034                                        } catch(OptionException e) {
1035                                                System.out.println("The specified graph appears not be a proper URL.");
1036                                                System.exit(0);
1037                                        }
1038
1039                                        LinkedList<String> defaultGraphURIs = new LinkedList<>();
1040                                        if(graph != null) {
1041                                                defaultGraphURIs.add(graph.toString());
1042                                        }
1043                                        SparqlEndpoint se = new SparqlEndpoint(endpoint, defaultGraphURIs, new LinkedList<>());
1044//                                      Path tempDirectory = Files.createTempDirectory("dllearner");
1045                                        String cacheDir = System.getProperty("java.io.tmpdir") + File.separator + "dl-learner";
1046                                        ks = new SparqlEndpointKS(se, cacheDir);
1047                                }
1048                                ks.init();
1049                        } catch (URISyntaxException e2) {
1050                                e2.printStackTrace();
1051                        }
1052
1053                        URI resourceURI = null;
1054                        try {
1055                                resourceURI = (URI) options.valueOf("resource");
1056                        } catch(OptionException e) {
1057                                System.out.println("The specified resource appears not be a proper URI.");
1058                                System.exit(0);
1059                        }
1060                        //set credentials if needed
1061                        if(options.has("username") && options.has("password")){
1062                                final String username = (String) options.valueOf("username");
1063                                final String password = (String) options.valueOf("password");
1064                                Authenticator.setDefault (new Authenticator() {
1065                                        @Override
1066                                        protected PasswordAuthentication getPasswordAuthentication() {
1067                                                return new PasswordAuthentication(username, password.toCharArray());
1068                                        }
1069                                });
1070                        }
1071
1072                        if(ks.isRemote()){
1073                                // sanity check that endpoint/graph returns at least one triple
1074                                String query = "SELECT * WHERE {?s ?p ?o} LIMIT 1";
1075                                SparqlQuery sq = new SparqlQuery(query, ks.getEndpoint());
1076                                try {
1077                                        ResultSet q = sq.send();
1078                                        while (q.hasNext()) {
1079                                                q.next();
1080                                        }
1081                                } catch(QueryExceptionHTTP e) {
1082                                        System.out.println("Endpoint not reachable (check spelling).");
1083                                        System.exit(0);
1084                                }
1085                        }
1086
1087                        // map resource to correct type
1088                        OWLEntity resource = null;
1089                        if(options.valueOf("resource") != null) {
1090                                resource = new SPARQLTasks(ks.getEndpoint()).guessResourceType(resourceURI.toString(), true);
1091                                if(resource == null) {
1092                                        throw new IllegalArgumentException("Could not determine the type (class, object property or data property) of input resource " + options.valueOf("resource")
1093                                                        + ". Enrichment only works for classes and properties.");
1094                                }
1095                        }
1096
1097                        boolean useInference = (Boolean) options.valueOf("i");
1098                        boolean iterativeMode = (Boolean) options.valueOf("iterative");
1099//                      boolean verbose = (Boolean) options.valueOf("v");
1100                        double threshold = (Double) options.valueOf("t");
1101                        int maxNrOfResults = (Integer) options.valueOf("l");
1102                        if(maxNrOfResults == -1){
1103                                maxNrOfResults = Integer.MAX_VALUE;
1104                        }
1105
1106                        int chunksize = (Integer) options.valueOf("chunksize");
1107                        int maxExecutionTimeInSeconds = (Integer) options.valueOf("maxExecutionTimeInSeconds");
1108                        boolean omitExistingAxioms = (Boolean) options.valueOf("omitExistingAxioms");
1109
1110                        // TODO: some handling for inaccessible files or overwriting existing files
1111                        File f = (File) options.valueOf("o");
1112
1113                        // if plain and file option is given, redirect System.out to a file
1114                        if(options.has("o") && (!options.has("f") || options.valueOf("f").equals("plain"))) {
1115                                 PrintStream printStream = new PrintStream(new FileOutputStream(f));
1116                                 System.setOut(printStream);
1117                        }
1118
1119                        //extract namespaces to which the analyzed entities will be restricted
1120                        List<String> allowedNamespaces = options.valuesOf(allowedNamespacesOption);
1121
1122                        //check which entity types we have to process
1123                        boolean processObjectProperties = (Boolean) options.valueOf("op");
1124                        boolean processDataProperties = (Boolean) options.valueOf("dp");
1125                        boolean processClasses = (Boolean) options.valueOf("cls");
1126
1127                        Enrichment e = new Enrichment(ks, resource, threshold, maxNrOfResults, useInference, false, chunksize, maxExecutionTimeInSeconds, omitExistingAxioms);
1128                        e.setAllowedNamespaces(allowedNamespaces);
1129                        e.setIterativeMode(iterativeMode);
1130                        e.setProcessObjectProperties(processObjectProperties);
1131                        e.setProcessDataProperties(processDataProperties);
1132                        e.setProcessClasses(processClasses);
1133                        e.start();
1134
1135                        // print output in correct format
1136                        if(options.has("f")) {
1137                                List<AlgorithmRun> runs = e.getAlgorithmRuns();
1138                                List<OWLAxiom> axioms = new LinkedList<>();
1139                                for(AlgorithmRun run : runs) {
1140                                        axioms.addAll(e.toRDF(run.getAxioms(), run.getAlgorithm(), run.getParameters(), ks));
1141                                }
1142                                Model model = e.getModel(axioms);
1143                                OutputStream os = options.has("o") ? new FileOutputStream((File)options.valueOf("o")) : System.out;
1144
1145                                if(options.valueOf("f").equals("turtle")) {
1146                                        if(options.has("o")) {
1147                                                model.write(new FileOutputStream(f), "TURTLE");
1148                                        } else {
1149                                                System.out.println("ENRICHMENT[");
1150                                                model.write(System.out, "TURTLE");
1151                                                System.out.println("]");
1152                                        }
1153                                } else if(options.valueOf("f").equals("rdf/xml")){
1154                                        if(options.has("o")) {
1155                                                model.write(new FileOutputStream(f), "RDF/XML");
1156                                        } else {
1157                                                System.out.println("ENRICHMENT[");
1158                                                model.write(System.out, "RDF/XML");
1159                                                System.out.println("]");
1160                                        }
1161                                } else if(options.valueOf("f").equals("n-triples")){
1162                                        if(options.has("o")) {
1163                                                model.write(new FileOutputStream(f), "N-TRIPLES");
1164                                        } else {
1165                                                System.out.println("ENRICHMENT[");
1166                                                model.write(System.out, "N-TRIPLES");
1167                                                System.out.println("]");
1168                                        }
1169                                }
1170                        }
1171                        //serialize ontology
1172                        if(options.has("s")){
1173                                File file = (File)options.valueOf("s");
1174                                try {
1175                                        OWLOntology ontology = e.getGeneratedOntology(options.has("a"));
1176                                        OutputStream os = new BufferedOutputStream(new FileOutputStream(file));
1177                                        OWLManager.createOWLOntologyManager().saveOntology(ontology, new RDFXMLDocumentFormat(), os);
1178                                } catch (OWLOntologyStorageException e1) {
1179                                        throw new Error("Could not save ontology.");
1180                                }
1181                        }
1182
1183                }
1184
1185        }
1186
1187        /** Whether the URL is a file in the local file system. */
1188        public static boolean isLocalFile(java.net.URL url) {
1189                String scheme = url.getProtocol();
1190                return "file".equalsIgnoreCase(scheme) && !hasHost(url);
1191        }
1192
1193        public static boolean hasHost(java.net.URL url) {
1194                String host = url.getHost();
1195                return host != null && !"".equals(host);
1196        }
1197
1198}