OSiDashboard/LuzzuDashboard-Wrapper-API/KnowledgeBase/knowledgeBase.js

//© 2019 Dublin City University, Trinity College Dublin. All rights reserved. This material may not be reproduced, displayed, modified or distributed without the express prior written permission of the copyright holder.
var path = require('path');
var fs = require('fs');
var cache = require('memory-cache');
const knowledgeBaseFile = __dirname + '/output';
const statusFile = __dirname + '/status/statusStore.json';
const axios = require('axios');
const queryString = require('query-string');
const knowledgeBaseModel = require(__dirname + '/model/knowledgeBaseModel.js');
var Promise = require('promise');
const headers = { 'Accept': 'application/sparql-results+json', 'Content-Type': 'application/x-www-form-urlencoded' };
axios.defaults.timeout = 5000;

const UndefinedClassesAndPropertiesMetricQuery = require(__dirname + '/sparql/UndefinedClassesAndPropertiesMetric');
const HumanReadableLabellingMetricQuery = require(__dirname + '/sparql/HumanReadableLabellingMetric');
const CompatibleDatatypeQuery = require(__dirname + '/sparql/CompatibleDatatype');
const EntitiesAsMembersOfDisjointClassesMetricQuery = require(__dirname + '/sparql/EntitiesAsMembersOfDisjointClassesMetric');
const MisplacedClassesOrPropertiesMetricQuery = require(__dirname + '/sparql/MisplacedClassesOrPropertiesMetric');
const MisusedOwlDatatypeOrObjectPropertiesMetricQuery = require(__dirname + '/sparql/MisusedOwlDatatypeOrObjectPropertiesMetric');
const UsageOfIncorrectDomainOrRangeDatatypesMetricQuery = require(__dirname + '/sparql/UsageOfIncorrectDomainOrRangeDatatypesMetric');
const ExtensionalConcisenessMetricQuery_FetchProblem = require(__dirname + '/sparql/ExtensionalConcisenessMetric_FetchProblem');
const ExtensionalConcisenessMetricQuery_FetchAllTriples  = require(__dirname + '/sparql/ExtensionalConcisenessMetric_FetchAllTriples');
const ExtensionalConcisenessMetricQuery_FindRelated = require(__dirname + '/sparql/ExtensionalConcisenessMetric_FindRelated');

class knowledgeBases {


  getKnowledgeBase(req, res) {
    const knowledgeBaseID = parseInt(req.query.knowledgeBaseID, 10);
    let knowledgeBase=cache.get(knowledgeBaseID);
	let kbStatus=cache.get(knowledgeBaseID+"_STATUS")
	if(knowledgeBase===null)
	{
        return res.status(404).send({
          status: 'false',
          message: 'Knowledge Base not Created',
        });
	}
	else
	{
		if(kbStatus==="COMPLETED")
		{
			return res.status(200).send(
			{knowledgeBase}
			);

		}
		else
		{
			return res.status(404).send({
			status: 'false',
			message: 'Knowledge Base not Created',
			});
		}
	}	
	

  }


  
  getKnowledgeBaseStatus(req, res) {

    const knowledgeBaseID = parseInt(req.query.knowledgeBaseID, 10);
    let knowledgeBase=cache.get(knowledgeBaseID);
	let kbStatus=cache.get(knowledgeBaseID+"_STATUS")

	if(knowledgeBase===null)
	{
		console.log(knowledgeBaseID + ": Is Null");
		return res.status(404).send({
        status: 'false',
        message: 'Knowledge Base Not Found'
      });
	}
	else
	{
		console.log(knowledgeBaseID + ":"+ knowledgeBase.length);
		if(kbStatus==="INPROGRESS")
		{
			let percentage = cache.get(knowledgeBaseID+"_PERCENTAGE");
			console.log("Percentage :"+ percentage );
			return res.status(200).send({
				knowledgeBaseID: knowledgeBaseID,
				status:  'INPROGRESS',
				percentage: percentage
			});
		}
		else if (kbStatus==="COMPLETED")
		{
			return res.status(200).send({
				knowledgeBaseID: knowledgeBaseID,
				status : 'COMPLETED'
			});
		}
		else
		{
			return res.status(404).send({
			status: 'false',
			message: 'Knowledge Base Not Found'
      });
		}
			
	}		
	
  }


  generateKnowledgeBase(req, res) {
    const metrics = req.body.metrics;
    //const metrics = JSON.parse(req.body.metrics);
    const qualityGraph = req.body.qualityGraph;
    const dataGraph = req.body.dataGraph;
    const tripleStoreEndPoint = req.body.tripleStoreEndPoint;
	const isSparqlEndPoint = req.body.isSparqlEndPoint;
	const lastAssessmentDate = req.body.lastAssessmentDate;
	const knowledgeBaseMapping = req.body.knowledgeBaseMapping;
	
    let queryResults = [];
    let datasetSource = 'GRAPH';
	if(isSparqlEndPoint==='true')
	{
		//console.log("IS SPARQL ENDPOINT");
		datasetSource='SERVICE';
	}
    let knowledgeBaseID = Math.floor(Date.now());
	console.log(knowledgeBaseID );
	cache.put(knowledgeBaseID,queryResults);
	cache.put(knowledgeBaseID+"_STATUS","INPROGRESS",42000000);
        cache.put(knowledgeBaseID+"_PERCENTAGE",0,42000000);

    const knowledgeBaseNew = {
      knowledgeBaseID: knowledgeBaseID
    };

    function setImmediatePromise() {
      return new Promise((resolve) => {
        setImmediate(() => resolve());
      });
    }
	
	

      console.log("Metrics : "+metrics.length);

      var buildResultArray = function (query) {
        return new Promise(function (resolve, reject) {
			//console.log("MAKING SPARQL CALL" );
          axios.post(tripleStoreEndPoint, queryString.stringify({ query: query }), headers)
            .then(response => {
				//console.log("SPARQL CALL MADE" );
              if (response) {
				  //console.log("RESPONSE RECEIDED" );
                if (response.status === 200) {
					//console.log("200 CODE" );
                  console.log("buildResultArray : "+response.data.results.bindings.length);
                  response.data.results.bindings.forEach(function (binding) {
                    let ProblematicThing = "";
                    let Subject = "";
                    let Predicate = "";
                    let Object = "";
                    let FailedMertic = "";
                    let Exception = "";
               
                    if (binding.Subject !== undefined) {
                      Subject = "<"+binding.Subject.value+">";
                    }
                    if (binding.Predicate !== undefined) {
                      Predicate = "<"+binding.Predicate.value+">";
                    }
                    if (binding.Object !== undefined) {
						
						if(binding.Object.type==="uri")
						{
							Object +="<"+binding.Object.value+">";

						}
						else
						{
							if(binding.Object["xml:lang"]!==undefined)
							{
								Object +="\"\"\""+binding.Object.value+"\"\"\"@" +binding.Object["xml:lang"];
							}
							else if(binding.Object["datatype"]!==undefined)
							{
								let value = binding.Object.value.replace(/(\"|\')/g, '\\$1');
								Object +="\"\"\""+value+"\"\"\""+"^^<" +binding.Object["datatype"]+">" ;
							}
							else{
								Object +="\"\"\""+binding.Object.value+"\"\"\"";
							}
							  	  
						}
					  
                    }
                    if (binding.FailedMertic !== undefined) {
                      //FailedMertic = binding.FailedMertic.value;
					  FailedMertic = binding.FailedMertic.value.split("#")[1];
                    }
                    if (binding.Exception !== undefined) {
                      //Exception = binding.Exception.value;
					  Exception = binding.Exception.value.split("#")[1];
                    }
					
					ProblematicThing = knowledgeBaseMapping[FailedMertic][Exception];
					
                    let temp = {
                      "ProblematicThing": ProblematicThing,
                      "Subject": Subject,
                      "Predicate": Predicate,
                      "Object": Object,
                      "FailedMertic": FailedMertic,
                      "Exception": Exception
                    }
                    queryResults.push(temp);
                  });
                }
				else
				{
					//console.log("NOT 200 CODE" );
					reject(error);
				}
				
              }
			  //console.log("RESOLVING" );
              resolve();
            })
            .catch(error => {
			  //console.log("REJECTING" );
              reject(error);
            });
        });
      }
	  
     var buildextensionalConcisenessMetricProblems = function (query) {
        return new Promise(function (resolve, reject) {
		  let queryResultsTemp=[];
          axios.post(tripleStoreEndPoint, queryString.stringify({ query: query }), headers)
            .then(response => {
              if (response) {
                if (response.status === 200) {

                  console.log("buildextensionalConcisenessMetricProblems : "+response.data.results.bindings.length);
                  response.data.results.bindings.forEach(function (binding) {
                    let ProblematicThing = "";
                    let Subject = "";
                    let Predicate = "";
                    let Object = "";
                    let FailedMertic = "";
                    let Exception = "";

                    if (binding.Subject !== undefined) {
                      Subject = "<"+binding.Subject.value+">";
                    }
					
                    if (binding.Predicate !== undefined) {
                      Predicate = "<"+binding.Predicate.value+">";
                    }

					
                    if (binding.FailedMertic !== undefined) {
                      //FailedMertic = binding.FailedMertic.value;
					  //FailedMertic = binding.FailedMertic.value.split("#")[1];
					  FailedMertic = "ExtensionalConcisenessMetric";
                    }
                    if (binding.Exception !== undefined) {
                      //Exception = binding.Exception.value;
					  Exception = binding.Exception.value.split("#")[1];
                    }
					
                    //ProblematicThing = knowledgeBaseMapping[FailedMertic][Exception];
					ProblematicThing = knowledgeBaseMapping["ExtensionalConcisenessMetric"]["ResourceReplica"];
                    

                    let temp = {
                      "ProblematicThing": ProblematicThing,
                      "Subject": Subject,
                      "Predicate": Predicate,
                      "Object": Object,
                      "FailedMertic": FailedMertic,
                      "Exception": Exception
                    }
                    queryResultsTemp.push(temp);
                  });
                }
              }
              resolve(queryResultsTemp);
            })
            .catch(error => {
              //reject(queryResultsTemp);
			  resolve(queryResultsTemp);
            });
        });
      }
	  
     var buildextensionalConcisenessAllTripleQuery = function (query) {
        return new Promise(function (resolve, reject) {
		  let triples="";
          axios.post(tripleStoreEndPoint, queryString.stringify({ query: query }), headers)
            .then(response => {
              if (response) {
                if (response.status === 200) {

                  console.log("buildextensionalConcisenessAllTripleQuery : "+response.data.results.bindings.length);
				  let length=response.data.results.bindings.length;
				  response.data.results.bindings.forEach(function (binding,index) 
				  {
					  if(index===0)
					  {
						  triples="?Resource ";
					  }

					  triples +="<"+binding.Predicate.value+"> ";
		  
					  if(binding.Object.type==="uri")
					  {
						  triples +="<"+binding.Object.value+"> ";

					  }
					  else
					  {
						  if(binding.Object["xml:lang"]!==undefined)
						  {
							  triples +="\"\"\""+binding.Object.value+"\"\"\"@" +binding.Object["xml:lang"]+" " ;
						  }
						  else if(binding.Object["datatype"]!==undefined)
						  {
							  let value = binding.Object.value.replace(/(\"|\')/g, '\\$1');
							  //triples += "\"s\" ";
							  triples +="\"\"\""+value+"\"\"\""+"^^<" +binding.Object["datatype"]+"> " ;
						  }
						  else{
							  triples +="\"\"\""+binding.Object.value+"\"\"\" ";
						  }
							  	  
					  }
					  if(!(index===length-1))
					  {
						  triples += "; ";
					  }
					  
				  });
       
                }
              }
              resolve(triples);
            })
            .catch(error => {
              //reject(triples);
			  resolve(triples);
            });
        });
      }
	  

    var buildextensionalConcisenessResultArray = function (query) {
        return new Promise(function (resolve, reject) {
		  let related=[];
          axios.post(tripleStoreEndPoint, queryString.stringify({ query: query }), headers)
            .then(response => {
              if (response) {
                if (response.status === 200) {

                  console.log("buildextensionalConcisenessResultArray : "+response.data.results.bindings.length);
				  let length=response.data.results.bindings.length;
				  response.data.results.bindings.forEach(function (binding,index) 
				  {
					  related.push(binding.Resource.value);
				  });
       
                }
              }
              resolve(related);
            })
            .catch(error => {
              //reject(related);
			  resolve(related);
            });
        });
      }

      var triggerActivity = async function () {
        for (const metric of metrics) {
          let query = "";
          switch (metric.metric) {
            case 'UndefinedClassesAndPropertiesMetric':
			console.log("UndefinedClassesAndPropertiesMetric:"+datasetSource);
              query = UndefinedClassesAndPropertiesMetricQuery.UndefinedClassesAndPropertiesMetric(qualityGraph, dataGraph, metric.observationURI,datasetSource);
				await buildResultArray(query);
			  break;
            case 'HumanReadableLabellingMetric':
			console.log("HumanReadableLabellingMetric:"+datasetSource);
              query = HumanReadableLabellingMetricQuery.HumanReadableLabellingMetric(qualityGraph, dataGraph, metric.observationURI,datasetSource);
              await buildResultArray(query);
			  break;
            case 'CompatibleDatatype':
			console.log("CompatibleDatatype:"+datasetSource);
              query = CompatibleDatatypeQuery.CompatibleDatatype(qualityGraph, dataGraph, metric.observationURI,datasetSource);
              await buildResultArray(query);
			  break;
            case 'EntitiesAsMembersOfDisjointClassesMetric':
			console.log("EntitiesAsMembersOfDisjointClassesMetric:"+datasetSource);
              query = EntitiesAsMembersOfDisjointClassesMetricQuery.EntitiesAsMembersOfDisjointClassesMetric(qualityGraph, dataGraph, metric.observationURI,datasetSource);
			  await buildResultArray(query);
			  break;
            case 'MisplacedClassesOrPropertiesMetric':
			console.log("MisplacedClassesOrPropertiesMetric:"+datasetSource);
              query = MisplacedClassesOrPropertiesMetricQuery.MisplacedClassesOrPropertiesMetric(qualityGraph, dataGraph, metric.observationURI,datasetSource);
			  await buildResultArray(query);
			  break;
            case 'MisusedOwlDatatypeOrObjectPropertiesMetric':
			console.log("MisusedOwlDatatypeOrObjectPropertiesMetric:"+datasetSource);
              query = MisusedOwlDatatypeOrObjectPropertiesMetricQuery.MisusedOwlDatatypeOrObjectPropertiesMetric(qualityGraph, dataGraph, metric.observationURI,datasetSource);
              await buildResultArray(query);
			  break;
            case 'UsageOfIncorrectDomainOrRangeDatatypesMetric':
			console.log("UsageOfIncorrectDomainOrRangeDatatypesMetric:"+datasetSource);
              query = UsageOfIncorrectDomainOrRangeDatatypesMetricQuery.UsageOfIncorrectDomainOrRangeDatatypesMetric(qualityGraph, dataGraph, metric.observationURI,datasetSource);
              await buildResultArray(query);
			  break;
			case 'ExtensionalConcisenessMetric':
			console.log("ExtensionalConcisenessMetric:"+datasetSource);
			  query = ExtensionalConcisenessMetricQuery_FetchProblem.ExtensionalConcisenessMetric(qualityGraph, dataGraph, metric.observationURI,datasetSource);
			  let queryResultsTemp = await buildextensionalConcisenessMetricProblems(query);
              //console.log(queryResultsTemp);
			  let alreadyFoundList = [];
			  let alreadFoundBoolean = false;
			  for(const problem of queryResultsTemp)
			  {
				  alreadFoundBoolean=false;
				  //console.log("PROBLEM : "+problem.Subject);
				  let subject = problem.Subject;
				  for(const inList of alreadyFoundList)
				  {
					  if(inList===subject)
					  {
						  alreadFoundBoolean=true;
					  }
				  }
				  if(!alreadFoundBoolean)
				  {
					query = ExtensionalConcisenessMetricQuery_FetchAllTriples.ExtensionalConcisenessMetric(qualityGraph, dataGraph, subject,datasetSource);
					let triples = await buildextensionalConcisenessAllTripleQuery(query);
					//console.log(triples);
					let relatedResourcesQuery=ExtensionalConcisenessMetricQuery_FindRelated.ExtensionalConcisenessMetric(triples, dataGraph, subject,datasetSource);
					let relatedResources=await buildextensionalConcisenessResultArray(relatedResourcesQuery);
					//console.log(relatedResources);
					
					for(const related of relatedResources)
					{
						let tempProblem = null;
						tempProblem=JSON.parse(JSON.stringify(problem));
						tempProblem["Object"]="<"+related+">"
						queryResults.push(tempProblem);
						alreadyFoundList.push("<"+related+">");
					}
				  }
				  
			  }

			  break;
            default:
			console.log("default:"+datasetSource);
              query = "";
			  break;
          }
          
        }
      }

      triggerActivity().then(async function () {
		//console.log("AFTER TRIGGER ACTIVITY");  
		console.log("ARRAY COUNT : "+queryResults.length);  
        //Loop through each array object
		var start = Date.now();
        for (const [outerIndex, outerResult] of queryResults.entries()) {
		  //let resourceRelatedExceptions = [];
		  //let problematicThingRelatedExceptions = [];
		  let SubjectRelatedException = [];
		  let PredicateRelatedException = [];
		  let ObjectRelatedException = [];
		  let tempArray = [];
		  let tempSubjectRelatedException = [];
		  let tempPredicateRelatedException = [];
		  let tempObjectRelatedException = [];
		  //console.log(outerIndex);
		  //console.log((outerIndex+1)/queryResults.length);
		  cache.del(knowledgeBaseID+"_PERCENTAGE");
		  cache.put(knowledgeBaseID+"_PERCENTAGE",(outerIndex+1)/queryResults.length,42000000);

          //console.log(outerIndex);
          //Add Await call to setImmediate to prevent thread blocking 
          await setImmediatePromise()
 		  
		  //Find Resource related Exceptions
		  tempArray = [];
          for (const [innerIndex, innerResult] of queryResults.entries()) {

            //if (outerResult.Subject === innerResult.Subject & outerResult.Exception !== innerResult.Exception) {
			if (outerResult.Subject === innerResult.Subject) {
				tempArray.push(innerResult.Exception);
            }

          }
          //resourceRelatedExceptions = [...new Set(tempArray.map((exception) => (exception)))];
          //queryResults[outerIndex].resourceRelatedExceptions = [...resourceRelatedExceptions];
		  SubjectRelatedException = [...new Set(tempArray.map((exception) => (exception)))];
          queryResults[outerIndex].SubjectRelatedException = [...SubjectRelatedException];
		  
		  //Find Problematic Thing Related Exceptions
		  //tempArray = [];
		  tempSubjectRelatedException = [];
		  tempPredicateRelatedException = [];
		  tempObjectRelatedException = [];
		  for(const outerResultProblematicThing of outerResult.ProblematicThing)
		  {
			for (const [innerIndex, innerResult] of queryResults.entries()) {

				//if (outerResult[outerResultProblematicThing] === innerResult[outerResultProblematicThing] & outerResult.Exception !== innerResult.Exception) {
				if (outerResult[outerResultProblematicThing] === innerResult[outerResultProblematicThing]) {
					//tempArray.push(innerResult.Exception);
					//SubjectRelatedExceptions is same as ResourceRelatedException
					//if(outerResultProblematicThing==="Subject")
					//{
						//tempSubjectRelatedException.push(innerResult.Exception);
					//}
					if(outerResultProblematicThing==="Predicate")
					{
						tempPredicateRelatedException.push(innerResult.Exception);
					}
					if(outerResultProblematicThing==="Object")
					{
						tempObjectRelatedException.push(innerResult.Exception);
					}
				}
			}
		  }
		  //problematicThingRelatedExceptions = [...new Set(tempArray.map((exception) => (exception)))];
          //queryResults[outerIndex].problematicThingRelatedExceptions = [...problematicThingRelatedExceptions];
	
		  //subjectRelatedException = [...new Set(tempSubjectRelatedException.map((exception) => (exception)))];
          //queryResults[outerIndex].subjectRelatedException = [...subjectRelatedException];

		  PredicateRelatedException = [...new Set(tempPredicateRelatedException.map((exception) => (exception)))];
          queryResults[outerIndex].PredicateRelatedException = [...PredicateRelatedException];

		  ObjectRelatedException = [...new Set(tempObjectRelatedException.map((exception) => (exception)))];
          queryResults[outerIndex].ObjectRelatedException = [...ObjectRelatedException];
	      //console.log(queryResults);
		  
		  /*
		  //For Future Purpose
		  queryResults[outerIndex].RootProblematicThing = "";
		  queryResults[outerIndex].RootCauseException = "";
		  //STORING TO DB
		  let knowledgeBaseDocumentToSave = new knowledgeBaseModel(queryResults[outerIndex]);
          let ObjectID = await knowledgeBaseDocumentToSave.save();
		  //console.log(ObjectID['_id']);
		  queryResults[outerIndex].ObjectID = ObjectID['_id'];
		  */
        }
		var end = Date.now();
		var elapsed = end - start;
		
		function millisToMinutesAndSeconds(millis) {
			var minutes = Math.floor(millis / 60000);
			var seconds = ((millis % 60000) / 1000).toFixed(0);
			return minutes + ":" + (seconds < 10 ? '0' : '') + seconds;
		}
		console.log(millisToMinutesAndSeconds(elapsed));
        //Update Status as Completed
		//console.log(queryResults)

		console.log("UPDATING CACHE");
		cache.del(knowledgeBaseID);
		cache.put(knowledgeBaseID,queryResults,42000000);
		cache.del(knowledgeBaseID+"_STATUS");
		cache.put(knowledgeBaseID+"_STATUS","COMPLETED",42000000);
		console.log("Generated");
		

      }
      ).catch(function(error){
		  console.log("ERROR_" + error);
		cache.del(knowledgeBaseID);
	  });

	return res.status(201).send(knowledgeBaseNew);

  }
}

const knowledgeBase = new knowledgeBases();
export default knowledgeBase;