Source code

001/**
002 * Copyright (C) 2007 - 2016, Jens Lehmann
003 *
004 * This file is part of DL-Learner.
005 *
006 * DL-Learner is free software; you can redistribute it and/or modify
007 * it under the terms of the GNU General Public License as published by
008 * the Free Software Foundation; either version 3 of the License, or
009 * (at your option) any later version.
010 *
011 * DL-Learner is distributed in the hope that it will be useful,
012 * but WITHOUT ANY WARRANTY; without even the implied warranty of
013 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
014 * GNU General Public License for more details.
015 *
016 * You should have received a copy of the GNU General Public License
017 * along with this program.  If not, see <http://www.gnu.org/licenses/>.
018 */
019package org.dllearner.kb.sparql.simple;
020
021import org.apache.jena.ontology.OntClass;
022import org.apache.jena.ontology.OntModel;
023import org.apache.jena.ontology.OntModelSpec;
024import org.apache.jena.rdf.model.*;
025import com.jamonapi.Monitor;
026import com.jamonapi.MonitorFactory;
027import org.dllearner.core.AbstractKnowledgeSource;
028import org.dllearner.core.ComponentAnn;
029import org.dllearner.core.ComponentInitException;
030import org.dllearner.core.annotations.OutVariable;
031import org.dllearner.core.config.ConfigOption;
032import org.dllearner.kb.OWLOntologyKnowledgeSource;
033import org.dllearner.utilities.OwlApiJenaUtils;
034import org.dllearner.utilities.analyse.TypeOntology;
035import org.semanticweb.owlapi.model.OWLOntology;
036import org.semanticweb.owlapi.model.OWLOntologyManager;
037import org.slf4j.Logger;
038import org.slf4j.LoggerFactory;
039
040import java.util.*;
041
042@ComponentAnn(name = "efficient SPARQL fragment extractor", shortName = "sparqls", version = 0.1)
043public class SparqlSimpleExtractor extends AbstractKnowledgeSource implements OWLOntologyKnowledgeSource{
044
045    @ConfigOption(description = "URL of the SPARQL endpoint", required = true)
046    private String endpointURL = null;
047    @OutVariable
048    private OntModel model = null;
049    @ConfigOption(description = "List of the instances to use", required = true)
050    private List<String> instances = null;
051    @ConfigOption(description = "Filter for the tbox, can use variable ?s, ?p amd ?o", required = false)
052    private String aboxfilter = null;
053    @ConfigOption(description = "Filter for the tbox, can use variable ?example and ?class", required = false)
054    private String tboxfilter = null;
055
056    @ConfigOption(description = "recursion depth", required = true)
057    private int recursionDepth = 0;
058
059    @ConfigOption(description = "default graph URI", required = true)
060    private String defaultGraphURI = null;
061
062    @ConfigOption(description = "Sparql Query", required = false)
063    private String sparqlQuery = null;
064    @ConfigOption(description = "List of Ontology Schema URLs", required = true)
065    private List<String> ontologySchemaUrls = null;
066
067    private SchemaIndexer indexer;
068
069    private static Logger log = LoggerFactory.getLogger(SparqlSimpleExtractor.class);
070
071    public SparqlSimpleExtractor() {
072        model = ModelFactory.createOntologyModel(OntModelSpec.OWL_DL_MEM);
073    }
074
075    public static void main(String[] args) throws ComponentInitException {
076        SparqlSimpleExtractor extractor = new SparqlSimpleExtractor();
077        extractor.setEndpointURL("http://live.dbpedia.org/sparql");
078        extractor.setRecursionDepth(1);
079        extractor.setDefaultGraphURI("http://dbpedia.org");
080        List<String> instances = new ArrayList<>(7);
081        instances.add("http://dbpedia.org/resource/Democritus");
082        instances.add("http://dbpedia.org/resource/Zeno_of_Elea");
083        instances.add("http://dbpedia.org/resource/Plato");
084        instances.add("http://dbpedia.org/resource/Socrates");
085        instances.add("http://dbpedia.org/resource/Archytas");
086        instances.add("http://dbpedia.org/resource/Pythagoras");
087        instances.add("http://dbpedia.org/resource/Philolaus");
088
089        extractor.setInstances(instances);
090        extractor.init();
091        List<String> individuals = new LinkedList<>();
092        individuals.add("People");
093        individuals.add("Animals");
094        extractor.setInstances(individuals);
095        // System.out.println(extractor.createQuery());
096    }
097
098    public Set<String> difference(Set<String> alreadyQueriedIndividuals, OntModel model) {
099        Set<String> candidates = new HashSet<>();
100        Set<String> result = new HashSet<>();
101        for (ResIterator it = model.listSubjects(); it.hasNext(); ) {
102            candidates.add(it.next().getURI());
103        }
104        for (NodeIterator it = model.listObjects(); it.hasNext(); ) {
105            RDFNode cur = it.next();
106            if (cur.isURIResource() && !cur.isAnon()) {
107                candidates.add(((Resource) cur).getURI());
108            }
109        }
110
111        for (String candidate : candidates) {
112            if (!alreadyQueriedIndividuals.contains(candidate)) {
113//                System.out.println(candidate);
114                result.add(candidate);
115            }
116        }
117
118        return result;
119    }
120
121    @Override
122    public void init() throws ComponentInitException {
123
124        if (endpointURL == null) {
125            throw new ComponentInitException(
126                    "Parameter endpoint URL is required");
127        }
128        if (instances == null) {
129            throw new ComponentInitException("Parameter instances is required");
130        }
131        if (recursionDepth == 0) {
132            throw new ComponentInitException(
133                    "A value bigger than 0 is required for parameter recursionDepth");
134        }
135        if (ontologySchemaUrls == null) {
136            throw new ComponentInitException(
137                    "An ontology schema OWLClassExpression file (ontologyFile) in RDF is required");
138        }
139
140        Monitor monComp = MonitorFactory.start("Simple SPARQL Component")
141                .start();
142        Monitor monIndexer = MonitorFactory.start("Schema Indexer").start();
143        indexer = new SchemaIndexer();
144        indexer.setOntologySchemaUrls(ontologySchemaUrls);
145        indexer.init();
146        monIndexer.stop();
147
148        TypeOntology typeOntology = new TypeOntology();
149
150        Monitor monQueryingABox;
151        QueryExecutor executor = new QueryExecutor();
152        String queryString;
153        Set<String> instancesSet = new HashSet<>(instances);
154        Set<String> alreadyQueried = new HashSet<>();
155        Monitor typizeModel;
156        if (sparqlQuery == null) {
157            ABoxQueryGenerator aGenerator = new ABoxQueryGenerator();
158            for (int i = 0; i < recursionDepth; i++) {
159                if (instancesSet.isEmpty()) {
160                    log.warn("no new instances found more recursions (recursion {} )  {} new instances", i,instancesSet.size());
161
162                }
163
164                log.info("processing (recursion  {} ) {} new instances",i,instancesSet.size());
165                queryString = aGenerator.createQuery(instancesSet, aboxfilter);
166//                System.out.println(queryString);
167                log.debug("SPARQL: {}", queryString);
168
169                monQueryingABox = MonitorFactory.start("ABox query time");
170                                try {
171                                        executor.executeQuery(queryString, endpointURL, model, defaultGraphURI);
172                                } catch (Exception e) {
173                                        e.printStackTrace();
174                                }
175                monQueryingABox.stop();
176
177                typizeModel=MonitorFactory.start("Typize the model");
178                model=typeOntology.addTypetoJena(model, instances, null);
179                typizeModel.stop();
180
181                alreadyQueried.addAll(instancesSet);
182                instancesSet = difference(alreadyQueried, model);
183
184            }
185
186            log.info("recursion depth: {} reached, {} new instances",recursionDepth,instancesSet.size());
187
188            //queryString = aGenerator.createLastQuery(instances, model, filters);
189            //log.debug("SPARQL: {}", queryString);
190
191            //monQueryingABox = MonitorFactory.start("ABox query time");
192            //Monitor monQueryingABox2 = MonitorFactory.start("ABox query time last query");
193            //executor.executeQuery(queryString, endpointURL, model, defaultGraphURI);
194            //monQueryingABox.stop();
195            //monQueryingABox2.stop();
196
197        } else {
198            monQueryingABox = MonitorFactory.getTimeMonitor("ABox query time").start();
199            executor.executeQuery(sparqlQuery, endpointURL, model, null);
200            monQueryingABox.stop();
201        }
202
203        TBoxQueryGenerator tGenerator = new TBoxQueryGenerator();
204
205        //TODO check if all instances are queried. model.listIndividuals().toSet()
206        queryString = tGenerator.createQuery(alreadyQueried, tboxfilter);
207
208        Monitor monQueryingTBox = MonitorFactory.start("TBox query time");
209
210        executor.executeQuery(queryString, endpointURL, model, defaultGraphURI);
211        monQueryingTBox.stop();
212
213        Monitor monIndexing = MonitorFactory.start("Querying index and conversion");
214        Set<OntClass> classes = model.listClasses().toSet();
215        for (OntClass ontClass : classes) {
216            OntModel hierarchy = indexer.getHierarchyForURI(ontClass.getURI());
217            if (hierarchy != null) {
218                model.add(hierarchy);
219                log.debug("{}", model);
220            }
221        }
222
223        monIndexing.stop();
224        monComp.stop();
225//        log.info("*******Simple SPARQL Extractor********");
226//        /*for (Monitor monitor : MonitorFactory.getRootMonitor().getMonitors()) {
227//            log.info("* {} *", monitor);
228//        }*/
229//        log.info(JamonMonitorLogger.getStringForAllSortedByLabel());
230//        log.info("**************************************");
231        
232        initialized = true;
233    }
234
235    public String getEndpointURL() {
236        return endpointURL;
237    }
238
239    public void setEndpointURL(String endpointURL) {
240        this.endpointURL = endpointURL;
241    }
242
243    public String getDefaultGraphURI() {
244        return defaultGraphURI;
245    }
246
247    public void setDefaultGraphURI(String defaultGraphURI) {
248        this.defaultGraphURI = defaultGraphURI;
249    }
250
251    public Model getModel() {
252        return model;
253    }
254
255    public void setModel(OntModel model) {
256        this.model = model;
257    }
258
259    public String getAboxfilter() {
260        return aboxfilter;
261    }
262
263    public void setAboxfilter(String aboxfilter) {
264        this.aboxfilter = aboxfilter;
265    }
266
267    /**
268     * @return the instances
269     */
270    public List<String> getInstances() {
271        return instances;
272    }
273
274    /**
275     * @param instances the instances to set
276     */
277    public void setInstances(List<String> instances) {
278        this.instances = instances;
279    }
280
281    /**
282     * @return the recursionDepth
283     */
284    public int getRecursionDepth() {
285        return recursionDepth;
286    }
287
288    /**
289     * @param recursionDepth the recursionDepth to set
290     */
291    public void setRecursionDepth(int recursionDepth) {
292        this.recursionDepth = recursionDepth;
293    }
294
295    public List<String> getOntologySchemaUrls() {
296        return ontologySchemaUrls;
297    }
298
299    public void setOntologySchemaUrls(List<String> ontologySchemaUrls) {
300        this.ontologySchemaUrls = ontologySchemaUrls;
301    }
302
303    public String getTboxfilter() {
304        return tboxfilter;
305    }
306
307    public void setTboxfilter(String tboxfilter) {
308        this.tboxfilter = tboxfilter;
309    }
310
311    public String getSparqlQuery() {
312        return sparqlQuery;
313    }
314
315    public void setSparqlQuery(String sparqlQuery) {
316        this.sparqlQuery = sparqlQuery;
317    }
318
319    @Override
320    public OWLOntology createOWLOntology(OWLOntologyManager manager) {
321        return OwlApiJenaUtils.getOWLOntology(model);
322    }
323
324}