001package org.dllearner.scripts;
002
003import com.clarkparsia.owlapi.explanation.PelletExplanation;
004import com.clarkparsia.owlapi.explanation.io.manchester.ManchesterSyntaxExplanationRenderer;
005import com.clarkparsia.pellet.owlapiv3.PelletReasoner;
006import com.clarkparsia.pellet.owlapiv3.PelletReasonerFactory;
007import org.apache.jena.query.*;
008import org.apache.jena.query.ResultSet;
009import org.apache.jena.rdf.model.Model;
010import org.apache.jena.rdf.model.ModelFactory;
011import org.apache.jena.vocabulary.OWL;
012import org.apache.jena.vocabulary.RDFS;
013import org.apache.commons.compress.compressors.CompressorException;
014import org.apache.commons.compress.compressors.CompressorStreamFactory;
015import org.apache.log4j.*;
016import org.dllearner.kb.SparqlEndpointKS;
017import org.dllearner.kb.sparql.*;
018import org.ini4j.IniPreferences;
019import org.ini4j.InvalidFileFormatException;
020import org.semanticweb.owlapi.apibinding.OWLManager;
021import org.semanticweb.owlapi.formats.RDFXMLDocumentFormat;
022import org.semanticweb.owlapi.model.*;
023import org.semanticweb.owlapi.model.parameters.Imports;
024import org.semanticweb.owlapi.reasoner.*;
025import uk.ac.manchester.cs.owl.owlapi.OWLDataFactoryImpl;
026
027import java.io.*;
028import java.net.URL;
029import java.sql.*;
030import java.util.*;
031import java.util.Map.Entry;
032import java.util.prefs.Preferences;
033
034//import org.dllearner.common.index.ModelGenerator;
035//import org.dllearner.common.index.ModelGenerator.Strategy;
036
037/**
038 * FIXME: I just commented out all lines that caused errors. So, this class
039 * won't work as expected!!!
040 *  
041 * @author patrick
042 */
043public class SPARQLSampleDebugging {
044        
045        private SparqlEndpoint endpoint;
046        private SparqlEndpointKS ks;
047        private ExtractionDBCache cache = new ExtractionDBCache("cache");
048        
049        private int sampleSize = 10;
050        private int depth = 3;
051        private int nrOfChunks = 1;
052        private int maxNrOfExplanations = 10;
053        
054        private Logger logger = Logger.getLogger(SPARQLSampleDebugging.class);
055        
056        private Logger functionalLogger = Logger.getLogger("functionality");
057        private Logger asymmetricLogger = Logger.getLogger("asymmetry");
058        private Logger irreflexiveLogger = Logger.getLogger("irreflexivity");
059        
060        private Connection conn;
061        private PreparedStatement ps;
062        
063        private OWLDataFactory factory = new OWLDataFactoryImpl();
064        
065        private OWLOntology dbpediaOntology;
066        private OWLReasoner dbpediaReasoner;
067        private OWLOntologyManager manager = OWLManager.createOWLOntologyManager();
068        
069        private Random randomGen = new Random(2222343);
070        
071        static {PelletExplanation.setup();}
072        
073        public SPARQLSampleDebugging(SparqlEndpoint endpoint) {
074                this.endpoint = endpoint;
075                this.ks = new SparqlEndpointKS(endpoint);
076                initDBConnection();
077                dbpediaOntology = loadDBpediaOntology();
078                dbpediaReasoner = PelletReasonerFactory.getInstance().createNonBufferingReasoner(dbpediaOntology);
079                dbpediaReasoner.precomputeInferences(InferenceType.CLASS_HIERARCHY);
080        }
081        
082        private void initDBConnection() {
083                try {
084                        String iniFile = "db_settings.ini";
085                        Preferences prefs = new IniPreferences(new FileReader(iniFile));
086                        String dbServer = prefs.node("database").get("server", null);
087                        String dbName = prefs.node("database").get("name", null);
088                        String dbUser = prefs.node("database").get("user", null);
089                        String dbPass = prefs.node("database").get("pass", null);
090
091                        Class.forName("com.mysql.jdbc.Driver");
092                        String url = "jdbc:mysql://" + dbServer + "/" + dbName;
093                        conn = DriverManager.getConnection(url, dbUser, dbPass);
094                        
095                        ps = conn.prepareStatement("INSERT INTO debugging_evaluation ("
096                                        + "resource, fragment_size , consistent, nr_of_justifications, justifications, justificationsObject) " + "VALUES(?,?,?,?,?,?)");
097
098                } catch (ClassNotFoundException | InvalidFileFormatException | SQLException e) {
099                        e.printStackTrace();
100                } catch (IOException e) {
101                        e.printStackTrace();
102                }
103        }
104        
105        private Set<OWLAxiom> getBlackList(){
106                Set<OWLAxiom> blacklist = new HashSet<>();
107                OWLAxiom ax = factory.getOWLSubClassOfAxiom(
108                                factory.getOWLObjectSomeValuesFrom(factory.getOWLObjectProperty(IRI.create("http://dbpedia.org/ontology/leaderName")), factory.getOWLClass(IRI.create("http://dbpedia.org/ontology/Person"))),
109                                factory.getOWLClass(IRI.create("http://dbpedia.org/ontology/Settlement")));
110                blacklist.add(ax);
111                ax = factory.getOWLSubClassOfAxiom(
112                                factory.getOWLObjectSomeValuesFrom(factory.getOWLObjectProperty(IRI.create("http://dbpedia.org/ontology/language")), factory.getOWLClass(IRI.create("http://dbpedia.org/ontology/Language"))),
113                                factory.getOWLClass(IRI.create("http://dbpedia.org/ontology/Work")));
114                blacklist.add(ax);
115                ax = factory.getOWLSubClassOfAxiom(
116                                factory.getOWLObjectSomeValuesFrom(factory.getOWLObjectProperty(IRI.create("http://dbpedia.org/ontology/officialLanguage")), factory.getOWLClass(IRI.create("http://dbpedia.org/ontology/Language"))),
117                                factory.getOWLClass(IRI.create("http://dbpedia.org/ontology/Country")));
118                blacklist.add(ax);
119                return blacklist;
120        }
121        
122        private void writeToDB(String resource, int fragementSize, boolean consistent, Set<Set<OWLAxiom>> explanations) {
123                try {
124                        ps.setString(1, resource);
125                        ps.setInt(2, fragementSize);
126                        ps.setBoolean(3, consistent);
127                        if(explanations == null){
128                                ps.setInt(4, 0);
129                                ps.setNull(5, Types.NULL);
130                                ps.setObject(6, Types.NULL);
131                        } else {
132                                ps.setInt(4, explanations.size());
133                                ps.setString(5, explanations.toString());
134                                try {
135                                        OWLOntologyManager man = OWLManager.createOWLOntologyManager();
136                                        OWLOntology ont = OWLManager.createOWLOntologyManager().createOntology();
137                                        for(Set<OWLAxiom> axioms : explanations){
138                                                man.addAxioms(ont, axioms);
139                                        }
140                                        Model model = convert(ont);
141                                        ByteArrayOutputStream baos = new ByteArrayOutputStream();
142                                        model.write(baos, "N-TRIPLE");
143                                        String modelStr = baos.toString("UTF-8");
144                                        ps.setClob(6, new StringReader(modelStr));
145                                } catch (UnsupportedEncodingException | OWLOntologyCreationException e) {
146                                        logger.error("ERROR", e);
147                                }
148                        }
149                        
150
151                        ps.executeUpdate();
152                } catch (SQLException e) {
153                        logger.error("Error while writing to DB.", e);
154                        e.printStackTrace();
155                }
156
157        }
158        
159        private Set<String> extractSampleResourcesChunked(int size){
160                logger.info("Extracting " + sampleSize + " sample resources...");
161                long startTime = System.currentTimeMillis();
162                Set<String> resources = new HashSet<>();
163                
164                String query = "SELECT COUNT(DISTINCT ?s) WHERE {?s a ?type}";
165                ResultSet rs = SparqlQuery.convertJSONtoResultSet(cache.executeSelectQuery(endpoint, query));
166                int max = rs.next().getLiteral(rs.getResultVars().get(0)).getInt();
167                
168                for(int i = 0; i < nrOfChunks; i++){
169                        int offset = (int)(Math.random() * max);
170                        offset = Math.min(offset, offset-(size/nrOfChunks));offset = 236242;
171                        
172                        query = String.format("SELECT DISTINCT ?s WHERE {?s a ?type} LIMIT %d OFFSET %d", (size/nrOfChunks), offset);
173                        logger.info(query);
174                        rs = SparqlQuery.convertJSONtoResultSet(cache.executeSelectQuery(endpoint, query));
175                        
176                        while(rs.hasNext()){
177                                resources.add(rs.next().getResource("s").getURI());
178                        }
179                }
180                
181                logger.info("...done in " + (System.currentTimeMillis()-startTime) + "ms.");
182                return resources;
183        }
184        
185        private String extractSampleResource(int maxOffset){
186                long startTime = System.currentTimeMillis();
187                
188                int random = randomGen.nextInt(maxOffset);
189                logger.info("Extracting sample resource (" + random + ")...");
190                
191                String query = String.format("SELECT DISTINCT ?s WHERE {?s a ?type} LIMIT 1 OFFSET %d", random);
192                ResultSet rs = SparqlQuery.convertJSONtoResultSet(cache.executeSelectQuery(endpoint, query));
193                String resource = rs.next().getResource("s").getURI();
194                
195                
196                logger.info("...done in " + (System.currentTimeMillis()-startTime) + "ms.");
197                return resource;
198        }
199        
200        private OWLOntology extractSampleModule(Set<String> resources){
201                logger.info("Extracting sample module...");
202                long startTime = System.currentTimeMillis();
203                ConciseBoundedDescriptionGenerator cbdGen = new ConciseBoundedDescriptionGeneratorImpl(ks.getQueryExecutionFactory());
204                Model model = ModelFactory.createDefaultModel();
205                for(String resource : resources){
206                        model.add(cbdGen.getConciseBoundedDescription(resource, depth));
207                }
208                logger.info("...done in " + (System.currentTimeMillis()-startTime) + "ms.");
209                return convert(model);
210                
211        }
212        
213        private OWLOntology extractSampleModule(String resource){
214                logger.info("Extracting sample module...");
215                long startTime = System.currentTimeMillis();
216                ConciseBoundedDescriptionGenerator cbdGen = new ConciseBoundedDescriptionGeneratorImpl(ks.getQueryExecutionFactory());
217                Model model = cbdGen.getConciseBoundedDescription(resource, 3);
218                OWLOntology data = convert(model);
219                logger.info("...done in " + (System.currentTimeMillis()-startTime) + "ms.");
220                return data;
221                
222        }
223        
224        private Set<Set<OWLAxiom>> computeExplanations(OWLOntology ontology){
225                logger.info("Computing explanations...");
226                long startTime = System.currentTimeMillis();
227                boolean useGlassBox = true;
228                PelletExplanation expGen = new PelletExplanation(ontology, useGlassBox);
229                Set<Set<OWLAxiom>> explanations = expGen.getInconsistencyExplanations(maxNrOfExplanations);
230                logger.info("...done in " + (System.currentTimeMillis()-startTime) + "ms.");
231                return explanations;
232        }
233        
234        private Set<Set<OWLAxiom>> computeExplanations(PelletReasoner reasoner) {
235                logger.info("Computing explanations...");
236                long startTime = System.currentTimeMillis();
237                PelletExplanation expGen = new PelletExplanation(reasoner);
238                Set<Set<OWLAxiom>> explanations = new HashSet<>(maxNrOfExplanations);
239                explanations = expGen.getInconsistencyExplanations(maxNrOfExplanations);
240                logger.info("...done in " + (System.currentTimeMillis()-startTime) + "ms.");
241                return explanations;
242        }
243        
244        private OWLOntology loadReferenceOntology() throws OWLOntologyCreationException{
245                long startTime = System.currentTimeMillis();
246                logger.info("Loading reference ontology...");
247                OWLOntology ontology = OWLManager.createOWLOntologyManager().loadOntologyFromOntologyDocument(
248                                getClass().getClassLoader().getResourceAsStream("dbpedia_0.75_no_datapropaxioms.owl"));
249                logger.info("...done in " + (System.currentTimeMillis()-startTime) + "ms.");
250                return ontology;
251        }
252        
253        private OWLOntology loadDBpediaOntology() {
254                long startTime = System.currentTimeMillis();
255                logger.info("Loading DBpedia reference ontology...");
256                OWLOntology ontology = null;
257                try {
258                        URL dbpediaURL = new URL("http://downloads.dbpedia.org/3.7/dbpedia_3.7.owl.bz2");
259                        InputStream is = dbpediaURL.openStream();
260                        is = new CompressorStreamFactory().createCompressorInputStream("bzip2", is);
261                        ontology = OWLManager.createOWLOntologyManager().loadOntologyFromOntologyDocument(is);
262                } catch (CompressorException | IOException | OWLOntologyCreationException e) {
263                        e.printStackTrace();
264                }
265                logger.info("...done in " + (System.currentTimeMillis()-startTime) + "ms.");
266                return ontology;
267        }
268        
269        
270        
271        private OWLOntology convert(Model model) {
272                ByteArrayOutputStream baos = new ByteArrayOutputStream();
273                model.write(baos, "N-TRIPLE");
274                ByteArrayInputStream bais = new ByteArrayInputStream(baos.toByteArray());
275                OWLOntologyManager manager = OWLManager.createOWLOntologyManager();
276                OWLOntology retOnt = null;
277                //noinspection EmptyCatchBlock
278                try {
279                        retOnt = manager.loadOntologyFromOntologyDocument(bais);
280                } catch (OWLOntologyCreationException e) {
281
282                }
283                return retOnt;
284        }
285        
286        private Model convert(OWLOntology ontology) {
287                Model model = ModelFactory.createDefaultModel();
288                ByteArrayInputStream bais = null;
289                try {
290                        OWLOntologyManager man = OWLManager.createOWLOntologyManager();
291                        ByteArrayOutputStream baos = new ByteArrayOutputStream();
292                        man.saveOntology(ontology, new RDFXMLDocumentFormat(), baos);
293                        bais = new ByteArrayInputStream(baos.toByteArray());
294                        model.read(bais, null);
295                } catch (OWLOntologyStorageException e) {
296                        e.printStackTrace();
297                } finally {
298                        try {
299                                if(bais != null){
300                                        bais.close();
301                                }
302                        } catch (IOException e) {
303                                e.printStackTrace();
304                        }
305                }
306                return model;
307        }
308        
309        
310        private Set<OWLObjectProperty> getUnsatisfiableObjectProperties(PelletReasoner reasoner){
311                SortedSet<OWLObjectProperty> properties = new TreeSet<>((o1, o2) -> {
312                        return o1.toString().compareTo(o2.toString());
313                });
314                OWLDataFactory f = OWLManager.createOWLOntologyManager().getOWLDataFactory();
315                ManchesterSyntaxExplanationRenderer renderer = new ManchesterSyntaxExplanationRenderer();
316                PrintWriter out = new PrintWriter( System.out );
317                for(OWLObjectProperty p : reasoner.getRootOntology().getObjectPropertiesInSignature()){
318                        boolean satisfiable = reasoner.isSatisfiable(f.getOWLObjectExactCardinality(1, p));
319                        if(!satisfiable){
320                                properties.add(p);
321//                              PelletExplanation expGen = new PelletExplanation(reasoner);
322//                              try {
323//                                      Set<Set<OWLAxiom>> explanations = expGen.getUnsatisfiableExplanations(f.getOWLObjectExactCardinality(1, p),1);
324//                                      System.out.println(explanations);
325//                                      renderer.startRendering( out );
326//                                      renderer.render(explanations);
327//                                      renderer.endRendering();
328//                              } catch (UnsupportedOperationException e) {
329//                                      e.printStackTrace();
330//                              } catch (OWLException e) {
331//                                      e.printStackTrace();
332//                              } catch (IOException e) {
333//                                      e.printStackTrace();
334//                              }
335                        }
336                }
337                
338                return properties;
339                
340        }
341        
342        private Set<OWLDataProperty> getUnsatisfiableDataProperties(PelletReasoner reasoner){
343                SortedSet<OWLDataProperty> properties = new TreeSet<>();
344                OWLDataFactory f = OWLManager.createOWLOntologyManager().getOWLDataFactory();
345                for(OWLDataProperty p : reasoner.getRootOntology().getDataPropertiesInSignature()){
346                        boolean satisfiable = reasoner.isSatisfiable(f.getOWLDataExactCardinality(1, p));
347                        if(!satisfiable){
348                                properties.add(p);
349                        }
350                }
351                return properties;
352                
353        }
354        
355        
356        public void computeSampleExplanations(OWLOntology reference, int nrOfExplanations) throws IOException{
357                Set<Set<OWLAxiom>> sampleExplanations = new HashSet<>();
358                manager = reference.getOWLOntologyManager();
359                manager.removeAxioms(reference, getBlackList());
360                
361                PelletReasoner reasoner = PelletReasonerFactory.getInstance().createNonBufferingReasoner(reference);
362                reasoner.precomputeInferences(InferenceType.CLASS_HIERARCHY);
363                Set<OWLClass> unsatisfiableClasses = reasoner.getUnsatisfiableClasses().getEntitiesMinusBottom();
364                logger.info("Unsatisfiable classes(" + unsatisfiableClasses.size() + "): " + unsatisfiableClasses);
365                Set<OWLObjectProperty> unsatisfiableObjectProperties = getUnsatisfiableObjectProperties(reasoner);
366                logger.info("Unsatisfiable object properties(" + unsatisfiableObjectProperties.size() + "): " + unsatisfiableObjectProperties);
367                Set<OWLDataProperty> unsatisfiableDataProperties = getUnsatisfiableDataProperties(reasoner);
368                logger.info("Unsatisfiable data properties(" + unsatisfiableDataProperties.size() + "): " + unsatisfiableDataProperties);
369                OWLOntology module;
370                reasoner.isConsistent();
371                OWLOntologyManager man = OWLManager.createOWLOntologyManager();
372                man.addOntologyChangeListener( reasoner );
373                Model model;
374                
375                FileWriter out = new FileWriter( "log/alljustifications" + System.currentTimeMillis() + ".txt" );
376                ManchesterSyntaxExplanationRenderer renderer = new ManchesterSyntaxExplanationRenderer();
377                renderer.startRendering(out );
378                
379                String query = "SELECT COUNT(DISTINCT ?s) WHERE {?s a ?type}";
380                ResultSet rs = SparqlQuery.convertJSONtoResultSet(cache.executeSelectQuery(endpoint, query));
381                int maxOffset = rs.next().getLiteral(rs.getResultVars().get(0)).getInt();
382                
383                while(sampleExplanations.size() < nrOfExplanations){
384                        String resource = extractSampleResource(maxOffset);//resource = "http://dbpedia.org/resource/Pigeon_%28company%29";
385                        logger.info("###################################################################");
386                        logger.info("Resource " + resource);//resource = "http://dbpedia.org/resource/The_Man_Who_Wouldn%27t_Die";
387                        module = extractSampleModule(resource);module.getOWLOntologyManager().removeAxioms(module, module.getAxioms(AxiomType.DATA_PROPERTY_ASSERTION));
388                        manager.addAxioms(reference, module.getABoxAxioms(Imports.INCLUDED));
389                        manager.removeAxioms(reference, reference.getAxioms(AxiomType.DATA_PROPERTY_ASSERTION));
390                        boolean isConsistent = reasoner.isConsistent();
391                        logger.info("Consistent: " + isConsistent);
392                        Set<Set<OWLAxiom>> explanations =  null;
393                        if(!isConsistent){
394                                explanations = new HashSet<>();
395                                try {
396                                        explanations.addAll(computeExplanations(reasoner));
397                                } catch (Exception e1) {
398                                        continue;
399                                }
400                                model = convert(reference);
401                                explanations.addAll(computeInconsistencyExplanationsByAsymmetryPattern(reference, model));
402                                explanations.addAll(computeInconsistencyExplanationsByIrreflexivityPattern(reference, model));
403                                explanations.addAll(computeInconsistencyExplanationsByFunctionalityPattern(reference, model));
404                                explanations.addAll(computeInconsistencyExplanationsByInverseFunctionalityPattern(reference, model));
405                                logger.info("Found " + explanations.size() + " explanations.");
406                                for(Set<OWLAxiom> exp : explanations){
407                                        logger.info(exp + "\n");
408                                        out.flush();
409                                        try {
410                                                renderer.render( Collections.singleton(exp) );
411                                        } catch (UnsupportedOperationException | OWLException e) {
412                                                e.printStackTrace();
413                                        }
414                                }
415                                boolean addSample = true;
416                                while(addSample){
417                                        int rnd = 0;
418                                        if(explanations.size() > 1){
419                                                rnd = new Random().nextInt(explanations.size()-1);
420                                        }
421                                        Set<OWLAxiom> sampleExplanation = new ArrayList<>(explanations).get(rnd);
422                                        if(!containsUnsatisfiableObjectProperty(sampleExplanation)){
423                                                sampleExplanations.add(sampleExplanation);
424                                                addSample = false;
425                                        }
426                                        
427                                }
428                                
429                                Map<AxiomType, Integer> axiomType2CountMap = new HashMap<>();
430                                for(Set<OWLAxiom> explanation : explanations){
431                                        for(OWLAxiom axiom : explanation){
432                                                Integer cnt = axiomType2CountMap.get(axiom.getAxiomType());
433                                                if(cnt == null){
434                                                        cnt = 0;
435                                                }
436                                                cnt = cnt + 1;
437                                                axiomType2CountMap.put(axiom.getAxiomType(), cnt);
438                                        }
439                                }
440                                logger.info("Axiom type count:");
441                                for(Entry<AxiomType, Integer> entry : axiomType2CountMap.entrySet()){
442                                        logger.info(entry.getKey() + "\t: " + entry.getValue());
443                                }
444                        }
445                        man.removeAxioms(reference, module.getABoxAxioms(Imports.INCLUDED));
446//                      writeToDB(resource, module.getLogicalAxiomCount(), isConsistent, explanations);
447                }
448                renderer.endRendering();
449                
450                FileWriter sampleOut = new FileWriter( "log/sample_justifications" + System.currentTimeMillis() + ".txt" );
451                ManchesterSyntaxExplanationRenderer sampleRenderer = new ManchesterSyntaxExplanationRenderer();
452                sampleRenderer.startRendering(sampleOut);
453                for(Set<OWLAxiom> exp : sampleExplanations){
454                        try {
455                                sampleRenderer.render(Collections.singleton(exp));
456                                
457                        } catch (UnsupportedOperationException | OWLException e) {
458                                e.printStackTrace();
459                        }
460                }
461                sampleRenderer.endRendering();
462                
463        }
464        
465        
466        public void runOptimized(OWLOntology reference) throws IOException{
467                PelletReasoner reasoner = PelletReasonerFactory.getInstance().createNonBufferingReasoner(reference);
468                reasoner.precomputeInferences(InferenceType.CLASS_HIERARCHY);
469                Set<OWLClass> unsatisfiableClasses = reasoner.getUnsatisfiableClasses().getEntitiesMinusBottom();
470                logger.info("Unsatisfiable classes(" + unsatisfiableClasses.size() + "): " + unsatisfiableClasses);
471                Set<OWLObjectProperty> unsatisfiableObjectProperties = getUnsatisfiableObjectProperties(reasoner);
472                logger.info("Unsatisfiable object properties(" + unsatisfiableObjectProperties.size() + "): " + unsatisfiableObjectProperties);
473                Set<OWLDataProperty> unsatisfiableDataProperties = getUnsatisfiableDataProperties(reasoner);
474                logger.info("Unsatisfiable data properties(" + unsatisfiableDataProperties.size() + "): " + unsatisfiableDataProperties);
475                OWLOntology module;
476                reasoner.isConsistent();
477                OWLOntologyManager man = OWLManager.createOWLOntologyManager();
478                man.addOntologyChangeListener( reasoner );
479                Set<String> resources = extractSampleResourcesChunked(sampleSize);
480                Model model;
481                FileWriter sampleOut = new FileWriter( "log/sample_justifications.txt" );
482                FileWriter out = new FileWriter( "log/alljustifications.txt" );
483                ManchesterSyntaxExplanationRenderer sampleRenderer = new ManchesterSyntaxExplanationRenderer();
484                ManchesterSyntaxExplanationRenderer renderer = new ManchesterSyntaxExplanationRenderer();
485                sampleRenderer.startRendering(sampleOut);
486                renderer.startRendering(out );
487                for(String resource : resources){
488                        logger.info("###################################################################");
489                        logger.info("Resource " + resource);//resource = "http://dbpedia.org/resource/The_Man_Who_Wouldn%27t_Die";
490                        module = extractSampleModule(resource);
491                        man.addAxioms(reference, module.getABoxAxioms(Imports.INCLUDED));
492                        man.removeAxioms(reference, reference.getAxioms(AxiomType.DATA_PROPERTY_ASSERTION));
493                        boolean isConsistent = reasoner.isConsistent();
494                        logger.info("Consistent: " + isConsistent);
495                        Set<Set<OWLAxiom>> explanations =  null;
496                        if(!isConsistent){
497                                explanations = new HashSet<>();
498                                //noinspection EmptyCatchBlock
499                                try {
500                                        explanations.addAll(computeExplanations(reasoner));
501                                } catch (Exception e1) {
502                                }
503                                model = convert(reference);
504                                explanations.addAll(computeInconsistencyExplanationsByAsymmetryPattern(reference, model));
505                                explanations.addAll(computeInconsistencyExplanationsByIrreflexivityPattern(reference, model));
506                                explanations.addAll(computeInconsistencyExplanationsByFunctionalityPattern(reference, model));
507                                explanations.addAll(computeInconsistencyExplanationsByInverseFunctionalityPattern(reference, model));
508                                logger.info("Found " + explanations.size() + " explanations.");
509                                for(Set<OWLAxiom> exp : explanations){
510                                        logger.info(exp + "\n");
511                                        out.flush();
512                                        try {
513                                                renderer.render( Collections.singleton(exp) );
514                                        } catch (UnsupportedOperationException | OWLException e) {
515                                                e.printStackTrace();
516                                        }
517                                }
518                                boolean writeSample = true;
519                                while(writeSample){
520                                        int rnd = new Random().nextInt(explanations.size()-1);
521                                        Set<OWLAxiom> sampleExplanation = new ArrayList<>(explanations).get(rnd);
522                                        if(!containsUnsatisfiableObjectProperty(sampleExplanation)){
523                                                try {
524                                                        sampleRenderer.render(Collections.singleton(sampleExplanation));
525                                                        writeSample = false;
526                                                } catch (UnsupportedOperationException | OWLException e) {
527                                                        e.printStackTrace();
528                                                }
529                                        }
530                                        
531                                }
532                                
533                                Map<AxiomType, Integer> axiomType2CountMap = new HashMap<>();
534                                for(Set<OWLAxiom> explanation : explanations){
535                                        for(OWLAxiom axiom : explanation){
536                                                Integer cnt = axiomType2CountMap.get(axiom.getAxiomType());
537                                                if(cnt == null){
538                                                        cnt = 0;
539                                                }
540                                                cnt = cnt + 1;
541                                                axiomType2CountMap.put(axiom.getAxiomType(), cnt);
542                                        }
543                                }
544                                logger.info("Axiom type count:");
545                                for(Entry<AxiomType, Integer> entry : axiomType2CountMap.entrySet()){
546                                        logger.info(entry.getKey() + "\t: " + entry.getValue());
547                                }
548                        }
549                        man.removeAxioms(reference, module.getABoxAxioms(Imports.INCLUDED));
550//                      writeToDB(resource, module.getLogicalAxiomCount(), isConsistent, explanations);
551                        break;
552                }
553                renderer.endRendering();
554                sampleRenderer.endRendering();
555                
556        }
557        
558        public void runPatternBasedDetection(){
559                Model model = ModelFactory.createDefaultModel();
560
561                //read schema
562                InputStream in = getClass().getClassLoader().getResourceAsStream("dbpedia_0.75_no_datapropertyaxioms.owl");
563                model.read(in, null);
564                
565                //read data
566//              ModelGenerator modelGen = new ModelGenerator(endpoint, cache);
567//              model.add(modelGen.createModel("http://dbpedia.org/resource/Leipzig", Strategy.CHUNKS, depth));
568
569                //query for conflicts
570                String queryString = "SELECT ?s WHERE {?type1 <" + OWL.disjointWith + "> ?type2. ?s a ?type1. ?s a ?type2.} LIMIT 1";
571                queryString = "SELECT ?s ?p ?type1 ?type2 WHERE {" +
572                                "?type1 <" + OWL.disjointWith + "> ?type2." +
573                                                "?p <" + RDFS.domain + "> ?type1. ?p <" + RDFS.domain + "> ?type2." +
574                                                " ?s ?p ?o1." +
575                                                " ?s ?p ?o2.} LIMIT 10";
576                Query query = QueryFactory.create(queryString) ;
577                try (QueryExecution qexec = QueryExecutionFactory.create(query, model)) {
578                        ResultSet results = qexec.execSelect();
579                        for (; results.hasNext(); ) {
580                                QuerySolution soln = results.nextSolution();
581                                for (String var : results.getResultVars()) {
582                                        System.out.print(soln.get(var) + "|");
583                                }
584                                System.out.println();
585                        }
586                }
587        }
588        
589        public void checkFunctionalityViolation(OWLOntology ontology){
590                        Set<String> properties = new TreeSet<>();
591                        for(OWLAxiom ax : ontology.getAxioms(AxiomType.FUNCTIONAL_OBJECT_PROPERTY)){
592                                OWLObjectProperty prop = ((OWLFunctionalObjectPropertyAxiom)ax).getProperty().asOWLObjectProperty();
593                                properties.add(prop.toStringID());
594                        }
595                        for(OWLAxiom ax : ontology.getAxioms(AxiomType.FUNCTIONAL_DATA_PROPERTY)){
596                                OWLDataProperty prop = ((OWLFunctionalDataPropertyAxiom)ax).getProperty().asOWLDataProperty();
597                                properties.add(prop.toStringID());
598                        }
599                        for(String prop : properties){
600                                String query = "SELECT * WHERE {?s <%s> ?o1. ?s <%s> ?o2. FILTER(?o1 != ?o2)} LIMIT 1".replaceAll("%s", prop);
601                                ResultSet rs = SparqlQuery.convertJSONtoResultSet(cache.executeSelectQuery(endpoint, query));
602                                while(rs.hasNext()){
603                                        QuerySolution qs = rs.next();
604                                        functionalLogger.info("********************************************************");
605                                        functionalLogger.info(prop);
606                                        functionalLogger.info(qs.get("s") + "-->" + qs.get("o1"));
607                                        functionalLogger.info(qs.get("s") + "-->" + qs.get("o2"));
608                                }
609                                
610                        }
611        }
612        
613        public void checkIrreflexivityViolation(OWLOntology ontology){
614                        Set<String> properties = new TreeSet<>();
615                        for(OWLAxiom ax : ontology.getAxioms(AxiomType.IRREFLEXIVE_OBJECT_PROPERTY)){
616                                OWLObjectProperty prop = ((OWLIrreflexiveObjectPropertyAxiom)ax).getProperty().asOWLObjectProperty();
617                                properties.add(prop.toStringID());
618                        }
619                        for(String prop : properties){
620                                String query = "SELECT * WHERE {?s <%s> ?s.} LIMIT 1".replaceAll("%s", prop);
621                                ResultSet rs = SparqlQuery.convertJSONtoResultSet(cache.executeSelectQuery(endpoint, query));
622                                while(rs.hasNext()){
623                                        QuerySolution qs = rs.next();
624                                        irreflexiveLogger.info("********************************************************");
625                                        irreflexiveLogger.info(prop);
626                                        irreflexiveLogger.info(qs.get("s"));
627                                }
628                                
629                        }
630        }
631        
632        public void checkAsymmetryViolation(OWLOntology ontology){
633                        Set<String> properties = new TreeSet<>();
634                        for(OWLAxiom ax : ontology.getAxioms(AxiomType.ASYMMETRIC_OBJECT_PROPERTY)){
635                                OWLObjectProperty prop = ((OWLAsymmetricObjectPropertyAxiom)ax).getProperty().asOWLObjectProperty();
636                                properties.add(prop.toStringID());
637                        }
638                        for(String prop : properties){
639                                String query = "SELECT * WHERE {?s <%s> ?o.?o <%s> ?s.FILTER(?s != ?o)} LIMIT 1".replaceAll("%s", prop);
640                                ResultSet rs = SparqlQuery.convertJSONtoResultSet(cache.executeSelectQuery(endpoint, query));
641                                while(rs.hasNext()){
642                                        QuerySolution qs = rs.next();
643                                        asymmetricLogger.info("********************************************************");
644                                        asymmetricLogger.info(prop);
645                                        asymmetricLogger.info(qs.get("s") + "<-->" + qs.get("o"));
646                                }
647                                
648                        }
649        }
650        
651        private Set<Set<OWLAxiom>> computeInconsistencyExplanationsByFunctionalityPattern(OWLOntology ontology, Model model){
652                Set<Set<OWLAxiom>> explanations = new HashSet<>();
653                
654                for(OWLObjectProperty prop : extractObjectProperties(AxiomType.FUNCTIONAL_OBJECT_PROPERTY, ontology)){
655                        OWLAxiom axiom = factory.getOWLFunctionalObjectPropertyAxiom(prop);
656                        String queryString = "SELECT DISTINCT * WHERE {?s <%s> ?o1. ?s <%s> ?o2. FILTER(?o1 != ?o2)}".replace("%s", prop.toStringID());
657                        Query query = QueryFactory.create(queryString) ;
658                        try (QueryExecution qexec = QueryExecutionFactory.create(query, model)) {
659                                ResultSet results = qexec.execSelect();
660                                for (; results.hasNext(); ) {
661                                        Set<OWLAxiom> explanation = new HashSet<>();
662                                        explanation.add(axiom);
663                                        QuerySolution qs = results.next();
664                                        OWLIndividual subject = factory.getOWLNamedIndividual(IRI.create(qs.getResource("s").getURI()));
665                                        OWLIndividual object1 = factory.getOWLNamedIndividual(IRI.create(qs.getResource("o1").getURI()));
666                                        OWLIndividual object2 = factory.getOWLNamedIndividual(IRI.create(qs.getResource("o2").getURI()));
667                                        OWLAxiom ax = factory.getOWLObjectPropertyAssertionAxiom(prop, subject, object1);
668                                        explanation.add(ax);
669                                        ax = factory.getOWLObjectPropertyAssertionAxiom(prop, subject, object2);
670                                        explanation.add(ax);
671                                        explanations.add(explanation);
672                                }
673                        }
674                }
675                return explanations;
676        }
677        
678        private Set<Set<OWLAxiom>> computeInconsistencyExplanationsByInverseFunctionalityPattern(OWLOntology ontology, Model model){
679                Set<Set<OWLAxiom>> explanations = new HashSet<>();
680                
681                for(OWLObjectProperty prop : extractObjectProperties(AxiomType.INVERSE_FUNCTIONAL_OBJECT_PROPERTY, ontology)){
682                        OWLAxiom axiom = factory.getOWLInverseFunctionalObjectPropertyAxiom(prop);
683                        String queryString = "SELECT DISTINCT * WHERE {?s1 <%s> ?o. ?s2 <%s> ?o. FILTER(?s1 != ?s2)}".replace("%s", prop.toStringID());
684                        Query query = QueryFactory.create(queryString) ;
685                        try (QueryExecution qexec = QueryExecutionFactory.create(query, model)) {
686                                ResultSet results = qexec.execSelect();
687                                for (; results.hasNext(); ) {
688                                        Set<OWLAxiom> explanation = new HashSet<>();
689                                        explanation.add(axiom);
690                                        QuerySolution qs = results.next();
691                                        OWLIndividual subject1 = factory.getOWLNamedIndividual(IRI.create(qs.getResource("s1").getURI()));
692                                        OWLIndividual subject2 = factory.getOWLNamedIndividual(IRI.create(qs.getResource("s2").getURI()));
693                                        OWLIndividual object = factory.getOWLNamedIndividual(IRI.create(qs.getResource("o").getURI()));
694                                        OWLAxiom ax = factory.getOWLObjectPropertyAssertionAxiom(prop, subject1, object);
695                                        explanation.add(ax);
696                                        ax = factory.getOWLObjectPropertyAssertionAxiom(prop, subject2, object);
697                                        explanation.add(ax);
698                                        explanations.add(explanation);
699                                }
700                        }
701                }
702                return explanations;
703        }
704        
705        private Set<Set<OWLAxiom>> computeInconsistencyExplanationsByAsymmetryPattern(OWLOntology ontology, Model model){
706                Set<Set<OWLAxiom>> explanations = new HashSet<>();
707                
708                for(OWLObjectProperty prop : extractObjectProperties(AxiomType.ASYMMETRIC_OBJECT_PROPERTY, ontology)){
709                        OWLAxiom axiom = factory.getOWLAsymmetricObjectPropertyAxiom(prop);
710                        String queryString = "SELECT * WHERE {?s <%s> ?o. ?o <%s> ?s. FILTER(?o != ?s)}".replace("%s", prop.toStringID());
711                        Query query = QueryFactory.create(queryString) ;
712                        try (QueryExecution qexec = QueryExecutionFactory.create(query, model)) {
713                                ResultSet results = qexec.execSelect();
714                                for (; results.hasNext(); ) {
715                                        Set<OWLAxiom> explanation = new HashSet<>();
716                                        explanation.add(axiom);
717                                        QuerySolution qs = results.nextSolution();
718                                        OWLIndividual subject = factory.getOWLNamedIndividual(IRI.create(qs.getResource("s").getURI()));
719                                        OWLIndividual object = factory.getOWLNamedIndividual(IRI.create(qs.getResource("o").getURI()));
720                                        OWLAxiom ax = factory.getOWLObjectPropertyAssertionAxiom(prop, subject, object);
721                                        explanation.add(ax);
722                                        ax = factory.getOWLObjectPropertyAssertionAxiom(prop, object, subject);
723                                        explanation.add(ax);
724                                        explanations.add(explanation);
725                                }
726                        }
727                }
728                return explanations;
729        }
730        
731        private Set<Set<OWLAxiom>> computeInconsistencyExplanationsByIrreflexivityPattern(OWLOntology ontology, Model model){
732                Set<Set<OWLAxiom>> explanations = new HashSet<>();
733                
734                for(OWLObjectProperty prop : extractObjectProperties(AxiomType.IRREFLEXIVE_OBJECT_PROPERTY, ontology)){
735                        OWLAxiom axiom = factory.getOWLIrreflexiveObjectPropertyAxiom(prop);
736                        String queryString = "SELECT * WHERE {?s <%s> ?s.}".replace("%s", prop.toStringID());
737                        Query query = QueryFactory.create(queryString) ;
738                        try (QueryExecution qexec = QueryExecutionFactory.create(query, model)) {
739                                ResultSet results = qexec.execSelect();
740                                for (; results.hasNext(); ) {
741                                        Set<OWLAxiom> explanation = new HashSet<>();
742                                        explanation.add(axiom);
743                                        QuerySolution qs = results.nextSolution();
744                                        OWLIndividual subject = factory.getOWLNamedIndividual(IRI.create(qs.getResource("s").getURI()));
745                                        OWLAxiom ax = factory.getOWLObjectPropertyAssertionAxiom(prop, subject, subject);
746                                        explanation.add(ax);
747                                        explanations.add(explanation);
748                                }
749                        }
750                }
751                return explanations;
752        }
753        
754        private SortedSet<OWLObjectProperty> extractObjectProperties(AxiomType<? extends OWLAxiom> axiomType, OWLOntology ontology){
755                SortedSet<OWLObjectProperty> properties = new TreeSet<>();
756                for(OWLAxiom ax : ontology.getAxioms(axiomType)){
757                        properties.add(((OWLObjectPropertyCharacteristicAxiom)ax).getProperty().asOWLObjectProperty());
758                }
759//              properties.retainAll(data.getObjectPropertiesInSignature());
760                return properties;
761        }
762        
763        private boolean isLearnedAxiom(OWLAxiom axiom){
764                return !dbpediaReasoner.isEntailed(axiom);
765        }
766        
767        private boolean containsUnsatisfiableObjectProperty(Set<OWLAxiom> justification){
768                boolean value = false;
769                
770                OWLReasoner reasoner = null;
771                try {
772                        OWLOntology ontology = manager.createOntology(justification);
773                        manager.removeAxioms(ontology, ontology.getABoxAxioms(Imports.INCLUDED));
774                        reasoner = PelletReasonerFactory.getInstance().createNonBufferingReasoner(ontology);
775                        for(OWLObjectProperty p : ontology.getObjectPropertiesInSignature()){
776                                boolean satisfiable = reasoner.isSatisfiable(factory.getOWLObjectExactCardinality(1, p));
777                                if(!satisfiable){
778                                        value = true;
779                                        break;
780                                }
781                        }
782                } catch (TimeOutException | OWLOntologyCreationException | ReasonerInterruptedException | InconsistentOntologyException | FreshEntitiesException | ClassExpressionNotInProfileException e) {
783                        e.printStackTrace();
784                }
785                reasoner.dispose();
786                return value;
787        }
788        
789        public void removeAxiomsWithNamespace(Set<String> namespaces){
790                
791        }
792        
793        /**
794         * @param args
795         * @throws Exception 
796         */
797        public static void main(String[] args) throws Exception {
798                Logger.getRootLogger().setLevel(Level.INFO);
799                Logger.getRootLogger().removeAllAppenders();
800                Logger.getRootLogger().addAppender(new ConsoleAppender(new SimpleLayout()));
801                Logger.getRootLogger().addAppender(new FileAppender(new SimpleLayout(), "log/debug.log"));
802                Logger.getLogger("functionality").addAppender(new FileAppender(new SimpleLayout(), "log/functionality_violations.log"));
803                Logger.getLogger("irreflexivity").addAppender(new FileAppender(new SimpleLayout(), "log/irreflexivity_violations.log"));
804                Logger.getLogger("asymmetry").addAppender(new FileAppender(new SimpleLayout(), "log/asymmetry_violations.log"));
805                Logger.getLogger(SPARQLSampleDebugging.class).setLevel(Level.INFO);
806                java.util.logging.Logger pelletLogger = java.util.logging.Logger.getLogger("com.clarkparsia.pellet");
807        pelletLogger.setLevel(java.util.logging.Level.OFF);
808        
809                SparqlEndpoint endpoint = new SparqlEndpoint(new URL("http://dbpedia.aksw.org:8902/sparql"),
810                                Collections.singletonList("http://dbpedia.org"), Collections.<String>emptyList());
811                
812                if(args.length != 1){
813                        System.out.println("Usage: SPARQLSampleDebugging <Schema-Ontology>");
814                        System.exit(0);
815                }
816                
817                InputStream is = new BufferedInputStream(new FileInputStream(args[0]));
818                if(args[0].endsWith("bz2")){
819                        is = new CompressorStreamFactory().createCompressorInputStream("bzip2", is);
820                }
821                OWLOntology ontology = OWLManager.createOWLOntologyManager().loadOntologyFromOntologyDocument(is);
822                
823                SPARQLSampleDebugging debug = new SPARQLSampleDebugging(endpoint);
824                
825//              debug.checkFunctionalityViolation(ontology);
826//              debug.checkAsymmetryViolation(ontology);
827//              debug.checkIrreflexivityViolation(ontology);
828                
829                long s1 = System.currentTimeMillis();
830                debug.computeSampleExplanations(ontology, 10);
831                long s2 = System.currentTimeMillis()-s1;
832                System.out.println(s2);
833                
834                /*long s3 = System.currentTimeMillis();
835                debug.run(ontology);
836                long s4 = System.currentTimeMillis()-s3;
837                
838                System.out.println(s2);
839                System.out.println(s4);*/
840
841        }
842
843}