Source code

001/**
002 * Copyright (C) 2007 - 2016, Jens Lehmann
003 *
004 * This file is part of DL-Learner.
005 *
006 * DL-Learner is free software; you can redistribute it and/or modify
007 * it under the terms of the GNU General Public License as published by
008 * the Free Software Foundation; either version 3 of the License, or
009 * (at your option) any later version.
010 *
011 * DL-Learner is distributed in the hope that it will be useful,
012 * but WITHOUT ANY WARRANTY; without even the implied warranty of
013 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
014 * GNU General Public License for more details.
015 *
016 * You should have received a copy of the GNU General Public License
017 * along with this program.  If not, see <http://www.gnu.org/licenses/>.
018 */
019package org.dllearner.algorithms.qtl.filters;
020
021import org.apache.jena.rdf.model.RDFNode;
022import org.apache.jena.rdf.model.Statement;
023import uk.ac.shef.wit.simmetrics.similaritymetrics.AbstractStringMetric;
024import uk.ac.shef.wit.simmetrics.similaritymetrics.JaroWinkler;
025import uk.ac.shef.wit.simmetrics.similaritymetrics.Levenshtein;
026import uk.ac.shef.wit.simmetrics.similaritymetrics.QGramsDistance;
027
028import java.util.*;
029import java.util.Map.Entry;
030import java.util.function.Predicate;
031
032public class KeywordBasedStatementFilter2 implements Predicate<Statement> {
033        
034        private Set<String> questionWords;
035        
036        private AbstractStringMetric qGramMetric;
037        private AbstractStringMetric levensteinMetric;
038        private AbstractStringMetric jaroWinklerMetric;
039        private I_Sub substringMetric;
040        
041        private double threshold = 0.4;
042        
043        private int topK = 3;
044        private double topKSumThreshold = 0.8;
045        
046        private Map<Statement, Double> statement2Similarity = new HashMap<>();
047        
048        private Map<RDFNode, Boolean> cache = new HashMap<>();
049        
050        private Map<Statement, String> statement2TokenMap = new HashMap<>();
051        
052        private Map<String, String> resource2TokenMap = new HashMap<>();
053        
054        int cnt = 0;
055        
056        public KeywordBasedStatementFilter2(Set<String> questionWords){
057                this.questionWords = questionWords;
058                qGramMetric = new QGramsDistance();
059                levensteinMetric = new Levenshtein();
060                jaroWinklerMetric = new JaroWinkler();
061                substringMetric = new I_Sub();
062                
063        }
064
065        private boolean isSimiliar2QuestionWord(String s, Statement st){
066                for(String word : questionWords){
067                        if(areSimiliar(word, s, st)){
068                                statement2TokenMap.put(st, word);
069                                resource2TokenMap.put(s, word);
070                                return true;
071                        }
072                }
073                return isSimilarWithSubstringMetrik(s, st);
074        }
075        
076        private boolean areSimiliar(String s1, String s2, Statement st){
077                return (qGramMetric.getSimilarity(s1, s2) >= threshold) || 
078                (levensteinMetric.getSimilarity(s1, s2) >= threshold);
079        }
080        
081        private boolean isSimilarWithSubstringMetrik(String s, Statement st){
082                SortedSet<Double> values = new TreeSet<>(Collections.reverseOrder());
083                for(String word : questionWords){
084                        double v = substringMetric.score(word, s, true);
085                        if(v >= threshold){statement2TokenMap.put(st, word);resource2TokenMap.put(s, word);
086                                return true;
087                        } else {
088                                values.add(v);
089                        }
090                } 
091                double sum = 0;
092                for(Double v : getTopK(values)){
093                        if(v >= 0){
094                                sum += v;
095                        }
096                        
097                }
098                if(sum >= topKSumThreshold){
099                        statement2TokenMap.put(st, "ALL");
100                }
101                return sum >= topKSumThreshold;
102        }
103        
104        private Set<Double> getTopK(SortedSet<Double> values){
105                Set<Double> top = new HashSet<>();
106                int k = 0;
107                for(Double v : values){
108                        if(k == topK){
109                                break;
110                        }
111                        top.add(v);
112                        k++;
113                }
114                return top;
115        }
116        
117        
118        private String getFragment(String uri){
119                int i = uri.lastIndexOf("#");
120                if(i > 0){
121                        return uri.substring(i+1);
122                } else {
123                        return uri.substring(uri.lastIndexOf("/")+1);
124                }
125        }
126
127        @Override
128        public boolean test(Statement s) {
129                Boolean similarPredicate = cache.get(s.getPredicate());
130                Boolean similarObject = cache.get(s.getObject());
131                if(similarPredicate != null && similarObject != null){
132                        String object = null;
133                        if(s.getObject().isURIResource()){
134                                object = getFragment(s.getObject().asResource().getURI());
135                        } else if(s.getObject().isLiteral()){
136                                object = s.getObject().asLiteral().getLexicalForm();
137                        }
138                        String token = resource2TokenMap.get(object);
139                        if(token != null){
140                                statement2TokenMap.put(s, token);
141                        } else {
142                                token = resource2TokenMap.get(getFragment(s.getPredicate().getURI()));
143                                if( token != null){
144                                        statement2TokenMap.put(s, token);
145                                } 
146                        }
147                
148                        
149                        return similarPredicate || similarObject;
150                } else if(similarPredicate == null && similarObject != null){
151                        if(similarObject){
152                                String object = null;
153                                if(s.getObject().isURIResource()){
154                                        object = getFragment(s.getObject().asResource().getURI());
155                                } else if(s.getObject().isLiteral()){
156                                        object = s.getObject().asLiteral().getLexicalForm();
157                                }
158                                String token = resource2TokenMap.get(object);
159                                if( token != null){
160                                        statement2TokenMap.put(s, token);
161                                }
162                                return true;
163                        } else {
164                                String predicate = getFragment(s.getPredicate().getURI());
165                                if (isSimiliar2QuestionWord(predicate, s)){
166                                        cache.put(s.getPredicate(), true);
167                                        return true;
168                                } else {
169                                        cache.put(s.getPredicate(), false);
170                                        return false;
171                                }
172                        }
173                } else if(similarPredicate != null && similarObject == null){
174                        if(similarPredicate){
175                                String object = null;
176                                if(s.getObject().isURIResource()){
177                                        object = getFragment(s.getObject().asResource().getURI());
178                                } else if(s.getObject().isLiteral()){
179                                        object = s.getObject().asLiteral().getLexicalForm();
180                                }
181                                if(isSimiliar2QuestionWord(object, s)){
182                                        cache.put(s.getObject(), true);
183                                        String token = resource2TokenMap.get(object);
184                                        if( token != null){
185                                                statement2TokenMap.put(s, token);
186                                        }
187                                        return true;
188                                } else {
189                                        cache.put(s.getObject(), false);
190                                }
191                                
192                                String token = resource2TokenMap.get(getFragment(s.getPredicate().getURI()));
193                                if( token != null){
194                                        statement2TokenMap.put(s, token);
195                                }
196                                return true;
197                        } else {
198                                String object = null;
199                                if(s.getObject().isURIResource()){
200                                        object = getFragment(s.getObject().asResource().getURI());
201                                } else if(s.getObject().isLiteral()){
202                                        object = s.getObject().asLiteral().getLexicalForm();
203                                }
204                                if(isSimiliar2QuestionWord(object, s)){
205                                        cache.put(s.getObject(), true);
206                                        return true;
207                                } else {
208                                        cache.put(s.getObject(), false);
209                                        return false;
210                                }
211                        }
212                } else {
213                        String object = null;
214                        if(s.getObject().isURIResource()){
215                                object = getFragment(s.getObject().asResource().getURI());
216                        } else if(s.getObject().isLiteral()){
217                                object = s.getObject().asLiteral().getLexicalForm();
218                        }
219                        if(isSimiliar2QuestionWord(object, s)){
220                                cache.put(s.getObject(), true);
221                                return true;
222                        } else {
223                                cache.put(s.getObject(), false);
224                        }
225                        
226                        String predicate = getFragment(s.getPredicate().getURI());
227                        if (isSimiliar2QuestionWord(predicate, s)){
228                                cache.put(s.getPredicate(), true);
229                                return true;
230                        } else {
231                                cache.put(s.getPredicate(), false);
232                        }
233                        return false;
234                }
235        }
236        
237//      @Override
238//      public boolean accept(Statement s) {
239//              String predicate = s.getPredicate().getURI().substring(s.getPredicate().getURI().lastIndexOf("/"));
240//              String object = null;
241//              if(s.getObject().isURIResource()){
242//                      object = s.getObject().asResource().getURI();
243//                      object = getFragment(s.getObject().asResource().getURI());
244//              } else if(s.getObject().isLiteral()){
245//                      object = s.getObject().asLiteral().getLexicalForm();
246//              }
247//              return isSimiliar2QuestionWord(predicate, s) || isSimiliar2QuestionWord(object, s);
248//      }
249        
250        public void setThreshold(double threshold){
251                this.threshold = threshold;
252        }
253        
254        public double getThreshold(){
255                return threshold;
256        }
257        
258        public Set<Statement> getStatementsBelowThreshold(double threshold){
259                Set<Statement> statements = new HashSet<>();
260                for(Entry<Statement, Double> entry : statement2Similarity.entrySet()){
261                        if(entry.getValue() < threshold){
262                                statements.add(entry.getKey());
263                        }
264                }
265                return statements;
266        }
267
268        public Map<Statement, String> getStatement2TokenMap() {
269                return statement2TokenMap;
270        }
271
272}