OSiDashboard/LuzzuDashboard-Wrapper-API/KnowledgeBase/knowledgeBase - Working.js

//© 2019 Dublin City University, Trinity College Dublin. All rights reserved. This material may not be reproduced, displayed, modified or distributed without the express prior written permission of the copyright holder.
var path = require('path');
var fs = require('fs');
var cache = require('memory-cache');
const knowledgeBaseFile = __dirname + '/output';
const statusFile = __dirname + '/status/statusStore.json';
const axios = require('axios');
const queryString = require('query-string');
const knowledgeBaseModel = require(__dirname + '/model/knowledgeBaseModel.js');
var Promise = require('promise');
const headers = { 'Accept': 'application/sparql-results+json', 'Content-Type': 'application/x-www-form-urlencoded' };

const UndefinedClassesAndPropertiesMetricQuery = require(__dirname + '/sparql/UndefinedClassesAndPropertiesMetric');
const HumanReadableLabellingMetricQuery = require(__dirname + '/sparql/HumanReadableLabellingMetric');
const CompatibleDatatypeQuery = require(__dirname + '/sparql/CompatibleDatatype');
const EntitiesAsMembersOfDisjointClassesMetricQuery = require(__dirname + '/sparql/EntitiesAsMembersOfDisjointClassesMetric');
const MisplacedClassesOrPropertiesMetricQuery = require(__dirname + '/sparql/MisplacedClassesOrPropertiesMetric');
const MisusedOwlDatatypeOrObjectPropertiesMetricQuery = require(__dirname + '/sparql/MisusedOwlDatatypeOrObjectPropertiesMetric');
const UsageOfIncorrectDomainOrRangeDatatypesMetricQuery = require(__dirname + '/sparql/UsageOfIncorrectDomainOrRangeDatatypesMetric');
const ExtensionalConcisenessMetricQuery_FetchProblem = require(__dirname + '/sparql/ExtensionalConcisenessMetric_FetchProblem');
const ExtensionalConcisenessMetricQuery_FetchAllTriples  = require(__dirname + '/sparql/ExtensionalConcisenessMetric_FetchAllTriples');
const ExtensionalConcisenessMetricQuery_FindRelated = require(__dirname + '/sparql/ExtensionalConcisenessMetric_FindRelated');

class knowledgeBases {


  getKnowledgeBase(req, res) {
    const knowledgeBaseID = parseInt(req.query.knowledgeBaseID, 10);
    let knowledgeBase=cache.get(knowledgeBaseID);
	
	if(knowledgeBase===null)
	{
        return res.status(404).send({
          status: 'false',
          message: 'Knowledge Base not Created',
        });
	}
	else
	{
		if(knowledgeBase.lenght===0)
		{
			
        return res.status(404).send({
          status: 'false',
          message: 'Knowledge Base not Created',
        });
		}
		else
		{
			return res.status(200).send(
			{knowledgeBase}
			);
		}
	}	
	

  }


  
  getKnowledgeBaseStatus(req, res) {

    const knowledgeBaseID = parseInt(req.query.knowledgeBaseID, 10);
    let knowledgeBase=cache.get(knowledgeBaseID);
	
	if(knowledgeBase===null)
	{
		console.log(knowledgeBaseID + ": Is Null");
		return res.status(404).send({
        status: 'false',
        message: 'Knowledge Base Not Found'
      });
	}
	else
	{
		console.log(knowledgeBaseID + ":"+ knowledgeBase.length);
		if(knowledgeBase.length===0)
		{
			
			return res.status(200).send({
				knowledgeBaseID: knowledgeBaseID,
				status:  'INPROGRESS'
			});
		}
		else
		{
			return res.status(200).send({
				knowledgeBaseID: knowledgeBaseID,
				status : 'COMPLETED'
			});
		}
	}		
	
  }


  generateKnowledgeBase(req, res) {
    const metrics = req.body.metrics;
    //const metrics = JSON.parse(req.body.metrics);
    const qualityGraph = req.body.qualityGraph;
    const dataGraph = req.body.dataGraph;
    const tripleStoreEndPoint = req.body.tripleStoreEndPoint;
	const isSparqlEndPoint = req.body.isSparqlEndPoint;
	const lastAssessmentDate = req.body.lastAssessmentDate;
	const knowledgeBaseMapping = req.body.knowledgeBaseMapping;
	
    let queryResults = [];
    let datasetSource = 'GRAPH';
	if(isSparqlEndPoint==='true')
	{
		console.log("IS SPARQL ENDPOINT");
		datasetSource='SERVICE';
	}
    let knowledgeBaseID = Math.floor(Date.now());
	cache.put(knowledgeBaseID,queryResults);
    const knowledgeBaseNew = {
      knowledgeBaseID: knowledgeBaseID
    };

    function setImmediatePromise() {
      return new Promise((resolve) => {
        setImmediate(() => resolve());
      });
    }
	
	

      console.log("Metrics : "+metrics.length);

      var buildResultArray = function (query) {
        return new Promise(function (resolve, reject) {

          axios.post(tripleStoreEndPoint, queryString.stringify({ query: query }), headers)
            .then(response => {
              if (response) {
                if (response.status === 200) {

                  console.log("buildResultArray : "+response.data.results.bindings.length);
                  response.data.results.bindings.forEach(function (binding) {
                    let ProblematicThing = "";
                    let Subject = "";
                    let Predicate = "";
                    let Object = "";
                    let FailedMertic = "";
                    let Exception = "";
               
                    if (binding.Subject !== undefined) {
                      Subject = "<"+binding.Subject.value+">";
                    }
                    if (binding.Predicate !== undefined) {
                      Predicate = "<"+binding.Predicate.value+">";
                    }
                    if (binding.Object !== undefined) {
						
						if(binding.Object.type==="uri")
						{
							Object +="<"+binding.Object.value+">";

						}
						else
						{
							if(binding.Object["xml:lang"]!==undefined)
							{
								Object +="\"\"\""+binding.Object.value+"\"\"\"@" +binding.Object["xml:lang"];
							}
							else if(binding.Object["datatype"]!==undefined)
							{
								let value = binding.Object.value.replace(/(\"|\')/g, '\\$1');
								Object +="\"\"\""+value+"\"\"\""+"^^<" +binding.Object["datatype"]+">" ;
							}
							else{
								Object +="\"\"\""+binding.Object.value+"\"\"\"";
							}
							  	  
						}
					  
                    }
                    if (binding.FailedMertic !== undefined) {
                      //FailedMertic = binding.FailedMertic.value;
					  FailedMertic = binding.FailedMertic.value.split("#")[1];
                    }
                    if (binding.Exception !== undefined) {
                      //Exception = binding.Exception.value;
					  Exception = binding.Exception.value.split("#")[1];
                    }
					
					ProblematicThing = knowledgeBaseMapping[FailedMertic][Exception];
					
                    let temp = {
                      "ProblematicThing": ProblematicThing,
                      "Subject": Subject,
                      "Predicate": Predicate,
                      "Object": Object,
                      "FailedMertic": FailedMertic,
                      "Exception": Exception
                    }
                    queryResults.push(temp);
                  });
                }
              }
              resolve();
            })
            .catch(error => {
              reject(error);
            });
        });
      }
	  
     var buildextensionalConcisenessMetricProblems = function (query) {
        return new Promise(function (resolve, reject) {
		  let queryResultsTemp=[];
          axios.post(tripleStoreEndPoint, queryString.stringify({ query: query }), headers)
            .then(response => {
              if (response) {
                if (response.status === 200) {

                  console.log("buildextensionalConcisenessMetricProblems : "+response.data.results.bindings.length);
                  response.data.results.bindings.forEach(function (binding) {
                    let ProblematicThing = "";
                    let Subject = "";
                    let Predicate = "";
                    let Object = "";
                    let FailedMertic = "";
                    let Exception = "";

                    if (binding.Subject !== undefined) {
                      Subject = "<"+binding.Subject.value+">";
                    }
					
                    if (binding.Predicate !== undefined) {
                      Predicate = "<"+binding.Predicate.value+">";
                    }

					
                    if (binding.FailedMertic !== undefined) {
                      //FailedMertic = binding.FailedMertic.value;
					  //FailedMertic = binding.FailedMertic.value.split("#")[1];
					  FailedMertic = "ExtensionalConcisenessMetric";
                    }
                    if (binding.Exception !== undefined) {
                      //Exception = binding.Exception.value;
					  Exception = binding.Exception.value.split("#")[1];
                    }
					
                    //ProblematicThing = knowledgeBaseMapping[FailedMertic][Exception];
					ProblematicThing = knowledgeBaseMapping["ExtensionalConcisenessMetric"]["ResourceReplica"];
                    

                    let temp = {
                      "ProblematicThing": ProblematicThing,
                      "Subject": Subject,
                      "Predicate": Predicate,
                      "Object": Object,
                      "FailedMertic": FailedMertic,
                      "Exception": Exception
                    }
                    queryResultsTemp.push(temp);
                  });
                }
              }
              resolve(queryResultsTemp);
            })
            .catch(error => {
              //reject(queryResultsTemp);
			  resolve(queryResultsTemp);
            });
        });
      }
	  
     var buildextensionalConcisenessAllTripleQuery = function (query) {
        return new Promise(function (resolve, reject) {
		  let triples="";
          axios.post(tripleStoreEndPoint, queryString.stringify({ query: query }), headers)
            .then(response => {
              if (response) {
                if (response.status === 200) {

                  console.log("buildextensionalConcisenessAllTripleQuery : "+response.data.results.bindings.length);
				  let length=response.data.results.bindings.length;
				  response.data.results.bindings.forEach(function (binding,index) 
				  {
					  if(index===0)
					  {
						  triples="?Resource ";
					  }

					  triples +="<"+binding.Predicate.value+"> ";
		  
					  if(binding.Object.type==="uri")
					  {
						  triples +="<"+binding.Object.value+"> ";

					  }
					  else
					  {
						  if(binding.Object["xml:lang"]!==undefined)
						  {
							  triples +="\"\"\""+binding.Object.value+"\"\"\"@" +binding.Object["xml:lang"]+" " ;
						  }
						  else if(binding.Object["datatype"]!==undefined)
						  {
							  let value = binding.Object.value.replace(/(\"|\')/g, '\\$1');
							  //triples += "\"s\" ";
							  triples +="\"\"\""+value+"\"\"\""+"^^<" +binding.Object["datatype"]+"> " ;
						  }
						  else{
							  triples +="\"\"\""+binding.Object.value+"\"\"\" ";
						  }
							  	  
					  }
					  if(!(index===length-1))
					  {
						  triples += "; ";
					  }
					  
				  });
       
                }
              }
              resolve(triples);
            })
            .catch(error => {
              //reject(triples);
			  resolve(triples);
            });
        });
      }
	  

    var buildextensionalConcisenessResultArray = function (query) {
        return new Promise(function (resolve, reject) {
		  let related=[];
          axios.post(tripleStoreEndPoint, queryString.stringify({ query: query }), headers)
            .then(response => {
              if (response) {
                if (response.status === 200) {

                  console.log("buildextensionalConcisenessResultArray : "+response.data.results.bindings.length);
				  let length=response.data.results.bindings.length;
				  response.data.results.bindings.forEach(function (binding,index) 
				  {
					  related.push(binding.Resource.value);
				  });
       
                }
              }
              resolve(related);
            })
            .catch(error => {
              //reject(related);
			  resolve(related);
            });
        });
      }

      var triggerActivity = async function () {
        for (const metric of metrics) {
          let query = "";
          switch (metric.metric) {
            case 'UndefinedClassesAndPropertiesMetric':
			console.log("UndefinedClassesAndPropertiesMetric:"+datasetSource);
              query = UndefinedClassesAndPropertiesMetricQuery.UndefinedClassesAndPropertiesMetric(qualityGraph, dataGraph, metric.observationURI,datasetSource);
				await buildResultArray(query);
			  break;
            case 'HumanReadableLabellingMetric':
			console.log("HumanReadableLabellingMetric:"+datasetSource);
              query = HumanReadableLabellingMetricQuery.HumanReadableLabellingMetric(qualityGraph, dataGraph, metric.observationURI,datasetSource);
              await buildResultArray(query);
			  break;
            case 'CompatibleDatatype':
			console.log("CompatibleDatatype:"+datasetSource);
              query = CompatibleDatatypeQuery.CompatibleDatatype(qualityGraph, dataGraph, metric.observationURI,datasetSource);
              await buildResultArray(query);
			  break;
            case 'EntitiesAsMembersOfDisjointClassesMetric':
			console.log("EntitiesAsMembersOfDisjointClassesMetric:"+datasetSource);
              query = EntitiesAsMembersOfDisjointClassesMetricQuery.EntitiesAsMembersOfDisjointClassesMetric(qualityGraph, dataGraph, metric.observationURI,datasetSource);
			  await buildResultArray(query);
			  break;
            case 'MisplacedClassesOrPropertiesMetric':
			console.log("MisplacedClassesOrPropertiesMetric:"+datasetSource);
              query = MisplacedClassesOrPropertiesMetricQuery.MisplacedClassesOrPropertiesMetric(qualityGraph, dataGraph, metric.observationURI,datasetSource);
			  await buildResultArray(query);
			  break;
            case 'MisusedOwlDatatypeOrObjectPropertiesMetric':
			console.log("MisusedOwlDatatypeOrObjectPropertiesMetric:"+datasetSource);
              query = MisusedOwlDatatypeOrObjectPropertiesMetricQuery.MisusedOwlDatatypeOrObjectPropertiesMetric(qualityGraph, dataGraph, metric.observationURI,datasetSource);
              await buildResultArray(query);
			  break;
            case 'UsageOfIncorrectDomainOrRangeDatatypesMetric':
			console.log("UsageOfIncorrectDomainOrRangeDatatypesMetric:"+datasetSource);
              query = UsageOfIncorrectDomainOrRangeDatatypesMetricQuery.UsageOfIncorrectDomainOrRangeDatatypesMetric(qualityGraph, dataGraph, metric.observationURI,datasetSource);
              await buildResultArray(query);
			  break;
			case 'ExtensionalConcisenessMetric':
			console.log("ExtensionalConcisenessMetric:"+datasetSource);
			  query = ExtensionalConcisenessMetricQuery_FetchProblem.ExtensionalConcisenessMetric(qualityGraph, dataGraph, metric.observationURI,datasetSource);
			  let queryResultsTemp = await buildextensionalConcisenessMetricProblems(query);
              //console.log(queryResultsTemp);
			  let alreadyFoundList = [];
			  let alreadFoundBoolean = false;
			  for(const problem of queryResultsTemp)
			  {
				  alreadFoundBoolean=false;
				  //console.log("PROBLEM : "+problem.Subject);
				  let subject = problem.Subject;
				  for(const inList of alreadyFoundList)
				  {
					  if(inList===subject)
					  {
						  alreadFoundBoolean=true;
					  }
				  }
				  if(!alreadFoundBoolean)
				  {
					query = ExtensionalConcisenessMetricQuery_FetchAllTriples.ExtensionalConcisenessMetric(qualityGraph, dataGraph, subject,datasetSource);
					let triples = await buildextensionalConcisenessAllTripleQuery(query);
					//console.log(triples);
					let relatedResourcesQuery=ExtensionalConcisenessMetricQuery_FindRelated.ExtensionalConcisenessMetric(triples, dataGraph, subject,datasetSource);
					let relatedResources=await buildextensionalConcisenessResultArray(relatedResourcesQuery);
					//console.log(relatedResources);
					
					for(const related of relatedResources)
					{
						let tempProblem = null;
						tempProblem=JSON.parse(JSON.stringify(problem));
						tempProblem["Object"]="<"+related+">"
						queryResults.push(tempProblem);
						alreadyFoundList.push("<"+related+">");
					}
				  }
				  
			  }

			  break;
            default:
			console.log("default:"+datasetSource);
              query = "";
			  break;
          }
          
        }
      }

      triggerActivity().then(async function () {
        //Loop through each array object
        for (const [outerIndex, outerResult] of queryResults.entries()) {
		  let resourceRelatedExceptions = [];
		  let problematicThingRelatedExceptions = [];
		  let tempArray = [];
		/*
          let SubjectRelatedExceptions = [];
          let PredicateRelatedExceptions = [];
          let ObjectRelatedExceptions = [];
          let SameTripleSameProblematicThingRelatedExceptions = [];
          let SameTripleDifferentProblematicThingRelatedExceptions = [];
        */  
          //console.log(outerIndex);
          //Add Await call to setImmediate to prevent thread blocking 
          await setImmediatePromise()
          //Find SubjectRelatedExceptions
		/*  
          tempArray = [];
          for (const [innerIndex, innerResult] of queryResults.entries()) {
            //Ignore same entry
            if (innerIndex !== outerIndex) {
              if (innerResult.ProblematicThing === "Subject" & outerResult.Subject === innerResult.Subject) {
                tempArray.push(innerResult.Exception);
              }
            }
          }
          SubjectRelatedExceptions = [...new Set(tempArray.map((exception) => (exception)))];
          queryResults[outerIndex].SubjectRelatedExceptions = [...SubjectRelatedExceptions];



          //Find PredicateRelatedExceptions
          tempArray = [];
          for (const [innerIndex, innerResult] of queryResults.entries()) {
            //Ignore same entry
            if (innerIndex !== outerIndex) {
              if (innerResult.ProblematicThing === "Predicate" & outerResult.Predicate === innerResult.Predicate) {
                tempArray.push(innerResult.Exception);
              }
            }
          }
          PredicateRelatedExceptions = [...new Set(tempArray.map((exception) => (exception)))];
          queryResults[outerIndex].PredicateRelatedExceptions = [...PredicateRelatedExceptions];



          //Find ObjectRelatedExceptions
          tempArray = [];
          for (const [innerIndex, innerResult] of queryResults.entries()) {
            //Ignore same entry
            if (innerIndex !== outerIndex) {
              if (innerResult.ProblematicThing === "Object" & outerResult.Object === innerResult.Object) {
                tempArray.push(innerResult.Exception);
              }
            }
          }
          ObjectRelatedExceptions = [...new Set(tempArray.map((exception) => (exception)))];
          queryResults[outerIndex].ObjectRelatedExceptions = [...ObjectRelatedExceptions];



          //Find SameTripleSameProblematicThingRelatedExceptions
          tempArray = [];
          for (const [innerIndex, innerResult] of queryResults.entries()) {
            //Ignore same entry
            if (innerIndex !== outerIndex) {
              if (innerResult.ProblematicThing === outerResult.ProblematicThing & outerResult.Subject === innerResult.Subject & outerResult.Precicate === innerResult.Precicate & outerResult.Object === innerResult.Object) {
                tempArray.push(innerResult.Exception);
              }
            }
          }
          SameTripleSameProblematicThingRelatedExceptions = [...new Set(tempArray.map((exception) => (exception)))];
          queryResults[outerIndex].SameTripleSameProblematicThingRelatedExceptions = [...SameTripleSameProblematicThingRelatedExceptions];



          //Find SameTripleDifferentProblematicThingRelatedExceptions
          tempArray = [];
          for (const [innerIndex, innerResult] of queryResults.entries()) {
            //Ignore same entry
            if (innerIndex !== outerIndex) {
              if (innerResult.ProblematicThing !== outerResult.ProblematicThing & outerResult.Subject === innerResult.Subject & outerResult.Precicate === innerResult.Precicate & outerResult.Object === innerResult.Object) {
                tempArray.push(innerResult.Exception);
              }
            }
          }
          SameTripleDifferentProblematicThingRelatedExceptions = [...new Set(tempArray.map((exception) => (exception)))];
          queryResults[outerIndex].SameTripleDifferentProblematicThingRelatedExceptions = [...SameTripleDifferentProblematicThingRelatedExceptions];
          //queryResults[outerIndex].knowledgeBaseID = knowledgeBaseID;
		*/
		
		   //Find Resource related Exceptions
		  tempArray = [];
          for (const [innerIndex, innerResult] of queryResults.entries()) {

            if (outerResult.Subject === innerResult.Subject & outerResult.Exception !== innerResult.Exception) {
				tempArray.push(innerResult.Exception);
            }

          }
          resourceRelatedExceptions = [...new Set(tempArray.map((exception) => (exception)))];
          queryResults[outerIndex].resourceRelatedExceptions = [...resourceRelatedExceptions];
		  
		  //Find Problematic Thing Related Exceptions
		  tempArray = [];
		  for(const outerResultProblematicThing of outerResult.ProblematicThing)
		  {
			for (const [innerIndex, innerResult] of queryResults.entries()) {

				if (outerResult[outerResultProblematicThing] === innerResult[outerResultProblematicThing] & outerResult.Exception !== innerResult.Exception) {
					tempArray.push(innerResult.Exception);
				}
			}
		  }
		  problematicThingRelatedExceptions = [...new Set(tempArray.map((exception) => (exception)))];
          queryResults[outerIndex].problematicThingRelatedExceptions = [...problematicThingRelatedExceptions];

        }
		

        //Update Status as Completed
		//console.log(queryResults)
		cache.del(knowledgeBaseID);
		cache.put(knowledgeBaseID,queryResults,60000);
		console.log("Generated");

      }
      ).catch(function(error){
		cache.del(knowledgeBaseID);
	  });

	return res.status(201).send(knowledgeBaseNew);

  }
}

const knowledgeBase = new knowledgeBases();
export default knowledgeBase;