OSiDashboard/LuzzuDashboard-Wrapper-API/KnowledgeBase/knowledgeBase - MongoDB.js

//© 2019 Dublin City University, Trinity College Dublin. All rights reserved. This material may not be reproduced, displayed, modified or distributed without the express prior written permission of the copyright holder.
var path = require('path');
var fs = require('fs');
const knowledgeBaseFile = __dirname + '/output';
const statusFile = __dirname + '/status/statusStore.json';
const axios = require('axios');
const queryString = require('query-string');
const knowledgeBaseModel = require(__dirname + '/model/knowledgeBaseModel.js');
var Promise = require('promise');
const headers = { 'Accept': 'application/sparql-results+json', 'Content-Type': 'application/x-www-form-urlencoded' };

const UndefinedClassesAndPropertiesMetricQuery = require(__dirname + '/sparql/UndefinedClassesAndPropertiesMetric');
const HumanReadableLabellingMetricQuery = require(__dirname + '/sparql/HumanReadableLabellingMetric');
const CompatibleDatatypeQuery = require(__dirname + '/sparql/CompatibleDatatype');
const EntitiesAsMembersOfDisjointClassesMetricQuery = require(__dirname + '/sparql/EntitiesAsMembersOfDisjointClassesMetric');
const MisplacedClassesOrPropertiesMetricQuery = require(__dirname + '/sparql/MisplacedClassesOrPropertiesMetric');
const MisusedOwlDatatypeOrObjectPropertiesMetricQuery = require(__dirname + '/sparql/MisusedOwlDatatypeOrObjectPropertiesMetric');
const UsageOfIncorrectDomainOrRangeDatatypesMetricQuery = require(__dirname + '/sparql/UsageOfIncorrectDomainOrRangeDatatypesMetric');
const ExtensionalConcisenessMetricQuery_FetchProblem = require(__dirname + '/sparql/ExtensionalConcisenessMetric_FetchProblem');
const ExtensionalConcisenessMetricQuery_FetchAllTriples  = require(__dirname + '/sparql/ExtensionalConcisenessMetric_FetchAllTriples');
const ExtensionalConcisenessMetricQuery_FindRelated = require(__dirname + '/sparql/ExtensionalConcisenessMetric_FindRelated');

class knowledgeBases {

	getKnowledgeBaseMongoDBStatus(req, res){
		return res.status(200).send({
        status: 'OK'
      });
	}

  getKnowledgeBase(req, res) {
    const fs = require('fs');
    let rawdata = fs.readFileSync(statusFile);
    let knowledgeBases = JSON.parse(rawdata);
    const knowledgeBaseID = parseInt(req.query.knowledgeBaseID, 10);
    let knowledgeBaseIndex;
    let knowledgeBaseStatus = "";
    knowledgeBases["knowledgeBases"].map((knowledgeBase, index) => {
      if (knowledgeBase.knowledgeBaseID === knowledgeBaseID) {
        knowledgeBaseIndex = index;
        knowledgeBaseStatus = knowledgeBase.status;
      }
    });
    if (knowledgeBaseIndex >= 0 & knowledgeBaseStatus === "COMPLETED") {
      knowledgeBaseModel.find({ knowledgeBaseID: knowledgeBases["knowledgeBases"][knowledgeBaseIndex].knowledgeBaseID }).then(function (knowledgeBaseDocument) {
        if (!knowledgeBaseDocument) {
          return res.status(404).send({
            status: 'false',
            message: 'Knowledge Base not Created',
          });
        }


        return res.status(200).send(
          knowledgeBaseDocument
        );
      }).catch(function (err) {
        return res.status(404).send({
          status: 'false',
          message: 'Knowledge Base not Created',
        });

      });

    }
    else {
      return res.status(404).send({
        status: 'false',
        message: 'Knowledge Base not Created',
      });
    }

  }


  findKnowledgeBaseDocumentIDs(req, res) {
    const fs = require('fs');
    let rawdata = fs.readFileSync(statusFile);
    let knowledgeBases = JSON.parse(rawdata);
    const knowledgeBaseID = parseInt(req.query.knowledgeBaseID, 10);
    let knowledgeBaseIndex;
    let knowledgeBaseStatus = "";
    knowledgeBases["knowledgeBases"].map((knowledgeBase, index) => {
      if (knowledgeBase.knowledgeBaseID === knowledgeBaseID) {
        knowledgeBaseIndex = index;
        knowledgeBaseStatus = knowledgeBase.status;
      }
    });
    if (knowledgeBaseIndex >= 0 & knowledgeBaseStatus === "COMPLETED") {
      knowledgeBaseModel.find({ knowledgeBaseID: knowledgeBases["knowledgeBases"][knowledgeBaseIndex].knowledgeBaseID }, '_id').then(function (knowledgeBaseDocument) {
        if (!knowledgeBaseDocument) {
          return res.status(404).send({
            status: 'false',
            message: 'No Knowledge Base Documents Available',
          });
        }


        return res.status(200).send(
          knowledgeBaseDocument
        );
      }).catch(function (err) {
        return res.status(404).send({
          status: 'false',
          message: 'No Knowledge Base Documents Available',
        });

      });

    }
    else {
      return res.status(404).send({
        status: 'false',
        message: 'No Knowledge Base Documents Available',
      });
    }

  }


  getKnowledgeBaseStatus(req, res) {
    const fs = require('fs');
    let rawdata = fs.readFileSync(statusFile);
    let knowledgeBases = JSON.parse(rawdata);
    const knowledgeBaseID = parseInt(req.query.knowledgeBaseID, 10);
    let knowledgeBaseIndex;
    knowledgeBases["knowledgeBases"].map((knowledgeBase, index) => {
      if (knowledgeBase.knowledgeBaseID === knowledgeBaseID) {
        knowledgeBaseIndex = index;
      }
    });
    if (knowledgeBaseIndex >= 0) {
      return res.status(200).send(
        knowledgeBases["knowledgeBases"][knowledgeBaseIndex]
      );
    }
    else {
      return res.status(404).send({
        status: 'false',
        message: 'Knowledge Base Not Found',
      });
    }
  }


  generateKnowledgeBase(req, res) {
    const fs = require('fs');
    const metrics = req.body.metrics;
    let rawdata = fs.readFileSync(statusFile);
    let knowledgeBase = JSON.parse(rawdata);
    //const metrics = JSON.parse(req.body.metrics);
    const qualityGraph = req.body.qualityGraph;
    const dataGraph = req.body.dataGraph;
    const tripleStoreEndPoint = req.body.tripleStoreEndPoint;
	const isSparqlEndPoint = req.body.isSparqlEndPoint;
	const lastAssessmentDate = req.body.lastAssessmentDate;
    let queryResults = [];
    let output = ''
    let knowledgeBaseID = Math.floor(Date.now());
    const knowledgeBaseNew = {
      knowledgeBaseID: knowledgeBaseID,
      status: "INPROGRESS",
	  lastAssessmentDate: lastAssessmentDate
    };

    function setImmediatePromise() {
      return new Promise((resolve) => {
        setImmediate(() => resolve());
      });
    }


    knowledgeBase["knowledgeBases"].push(knowledgeBaseNew);
    let updatedKnowledgeBase = JSON.stringify(knowledgeBase, null, 2);
    fs.writeFile(statusFile, updatedKnowledgeBase, function (error) {
      if (error) {
        //console.log('error writing file: ' + err);
        return res.status(500).send(
          {
            knowledgeBaseID: knowledgeBaseID,
            status: "FAILED"
          });
      }
      console.log("Metrics : "+metrics.length);

      var buildResultArray = function (query) {
        return new Promise(function (resolve, reject) {

          axios.post(tripleStoreEndPoint, queryString.stringify({ query: query }), headers)
            .then(response => {
              if (response) {
                if (response.status === 200) {

                  console.log("buildResultArray : "+response.data.results.bindings.length);
                  response.data.results.bindings.forEach(function (binding) {
                    let ProblematicThing = "";
                    let Subject = "";
                    let Predicate = "";
                    let Object = "";
                    let FailedMertic = "";
                    let Exception = "";
                    if (binding.ProblematicThing !== undefined) {
                      ProblematicThing = binding.ProblematicThing.value;
                    }
                    if (binding.Subject !== undefined) {
                      Subject = "<"+binding.Subject.value+">";
                    }
                    if (binding.Predicate !== undefined) {
                      Predicate = "<"+binding.Predicate.value+">";
                    }
                    if (binding.Object !== undefined) {
						
						if(binding.Object.type==="uri")
						{
							Object +="<"+binding.Object.value+">";

						}
						else
						{
							if(binding.Object["xml:lang"]!==undefined)
							{
								Object +="\"\"\""+binding.Object.value+"\"\"\"@" +binding.Object["xml:lang"];
							}
							else if(binding.Object["datatype"]!==undefined)
							{
								let value = binding.Object.value.replace(/(\"|\')/g, '\\$1');
								Object +="\"\"\""+value+"\"\"\""+"^^<" +binding.Object["datatype"]+">" ;
							}
							else{
								Object +="\"\"\""+binding.Object.value+"\"\"\"";
							}
							  	  
						}
					  
                    }
                    if (binding.FailedMertic !== undefined) {
                      //FailedMertic = binding.FailedMertic.value;
					  FailedMertic = binding.FailedMertic.value.split("#")[1];
                    }
                    if (binding.Exception !== undefined) {
                      //Exception = binding.Exception.value;
					  Exception = binding.Exception.value.split("#")[1];
                    }
                    let temp = {
                      "ProblematicThing": ProblematicThing,
                      "Subject": Subject,
                      "Predicate": Predicate,
                      "Object": Object,
                      "FailedMertic": FailedMertic,
                      "Exception": Exception
                    }
                    queryResults.push(temp);
                  });
                }
              }
              resolve();
            })
            .catch(error => {
              reject(error);
            });
        });
      }
	  
     var buildextensionalConcisenessMetricProblems = function (query) {
        return new Promise(function (resolve, reject) {
		  let queryResultsTemp=[];
          axios.post(tripleStoreEndPoint, queryString.stringify({ query: query }), headers)
            .then(response => {
              if (response) {
                if (response.status === 200) {

                  console.log("buildextensionalConcisenessMetricProblems : "+response.data.results.bindings.length);
                  response.data.results.bindings.forEach(function (binding) {
                    let ProblematicThing = "";
                    let Subject = "";
                    let Predicate = "";
                    let Object = "";
                    let FailedMertic = "";
                    let Exception = "";
                    if (binding.ProblematicThing !== undefined) {
                      ProblematicThing = binding.ProblematicThing.value;
                    }
                    if (binding.Subject !== undefined) {
                      Subject = "<"+binding.Subject.value+">";
                    }
					
                    if (binding.Predicate !== undefined) {
                      Predicate = "<"+binding.Predicate.value+">";
                    }

					
                    if (binding.FailedMertic !== undefined) {
                      //FailedMertic = binding.FailedMertic.value;
					  FailedMertic = binding.FailedMertic.value.split("#")[1];
                    }
                    if (binding.Exception !== undefined) {
                      //Exception = binding.Exception.value;
					  Exception = binding.Exception.value.split("#")[1];
                    }

                    let temp = {
                      "ProblematicThing": ProblematicThing,
                      "Subject": Subject,
                      "Predicate": Predicate,
                      "Object": Object,
                      "FailedMertic": FailedMertic,
                      "Exception": Exception
                    }
                    queryResultsTemp.push(temp);
                  });
                }
              }
              resolve(queryResultsTemp);
            })
            .catch(error => {
              //reject(queryResultsTemp);
			  resolve(queryResultsTemp);
            });
        });
      }
	  
     var buildextensionalConcisenessAllTripleQuery = function (query) {
        return new Promise(function (resolve, reject) {
		  let triples="";
          axios.post(tripleStoreEndPoint, queryString.stringify({ query: query }), headers)
            .then(response => {
              if (response) {
                if (response.status === 200) {

                  console.log("buildextensionalConcisenessAllTripleQuery : "+response.data.results.bindings.length);
				  let length=response.data.results.bindings.length;
				  response.data.results.bindings.forEach(function (binding,index) 
				  {
					  if(index===0)
					  {
						  triples="?Resource ";
					  }

					  triples +="<"+binding.Predicate.value+"> ";
		  
					  if(binding.Object.type==="uri")
					  {
						  triples +="<"+binding.Object.value+"> ";

					  }
					  else
					  {
						  if(binding.Object["xml:lang"]!==undefined)
						  {
							  triples +="\"\"\""+binding.Object.value+"\"\"\"@" +binding.Object["xml:lang"]+" " ;
						  }
						  else if(binding.Object["datatype"]!==undefined)
						  {
							  let value = binding.Object.value.replace(/(\"|\')/g, '\\$1');
							  //triples += "\"s\" ";
							  triples +="\"\"\""+value+"\"\"\""+"^^<" +binding.Object["datatype"]+"> " ;
						  }
						  else{
							  triples +="\"\"\""+binding.Object.value+"\"\"\" ";
						  }
							  	  
					  }
					  if(!(index===length-1))
					  {
						  triples += "; ";
					  }
					  
				  });
       
                }
              }
              resolve(triples);
            })
            .catch(error => {
              //reject(triples);
			  resolve(triples);
            });
        });
      }
	  

    var buildextensionalConcisenessResultArray = function (query) {
        return new Promise(function (resolve, reject) {
		  let related=[];
          axios.post(tripleStoreEndPoint, queryString.stringify({ query: query }), headers)
            .then(response => {
              if (response) {
                if (response.status === 200) {

                  console.log("buildextensionalConcisenessResultArray : "+response.data.results.bindings.length);
				  let length=response.data.results.bindings.length;
				  response.data.results.bindings.forEach(function (binding,index) 
				  {
					  related.push(binding.Resource.value);
				  });
       
                }
              }
              resolve(related);
            })
            .catch(error => {
              //reject(related);
			  resolve(related);
            });
        });
      }

      var triggerActivity = async function () {
        for (const metric of metrics) {
          let query = "";
          switch (metric.metric) {
            case 'UndefinedClassesAndPropertiesMetric':
			console.log("UndefinedClassesAndPropertiesMetric");
              query = UndefinedClassesAndPropertiesMetricQuery.UndefinedClassesAndPropertiesMetric(qualityGraph, dataGraph, metric.observationURI);
				await buildResultArray(query);
			  break;
            case 'HumanReadableLabellingMetric':
			console.log("HumanReadableLabellingMetric");
              query = HumanReadableLabellingMetricQuery.HumanReadableLabellingMetric(qualityGraph, dataGraph, metric.observationURI);
              await buildResultArray(query);
			  break;
            case 'CompatibleDatatype':
			console.log("CompatibleDatatype");
              query = CompatibleDatatypeQuery.CompatibleDatatype(qualityGraph, dataGraph, metric.observationURI);
              await buildResultArray(query);
			  break;
            case 'EntitiesAsMembersOfDisjointClassesMetric':
			console.log("EntitiesAsMembersOfDisjointClassesMetric");
              query = EntitiesAsMembersOfDisjointClassesMetricQuery.EntitiesAsMembersOfDisjointClassesMetric(qualityGraph, dataGraph, metric.observationURI);
              await buildResultArray(query);
			  break;
            case 'MisplacedClassesOrPropertiesMetric':
			console.log("MisplacedClassesOrPropertiesMetric");
              query = MisplacedClassesOrPropertiesMetricQuery.MisplacedClassesOrPropertiesMetric(qualityGraph, dataGraph, metric.observationURI);
              await buildResultArray(query);
			  break;
            case 'MisusedOwlDatatypeOrObjectPropertiesMetric':
			console.log("MisusedOwlDatatypeOrObjectPropertiesMetric");
              query = MisusedOwlDatatypeOrObjectPropertiesMetricQuery.MisusedOwlDatatypeOrObjectPropertiesMetric(qualityGraph, dataGraph, metric.observationURI);
              await buildResultArray(query);
			  break;
            case 'UsageOfIncorrectDomainOrRangeDatatypesMetric':
			console.log("UsageOfIncorrectDomainOrRangeDatatypesMetric");
              query = UsageOfIncorrectDomainOrRangeDatatypesMetricQuery.UsageOfIncorrectDomainOrRangeDatatypesMetric(qualityGraph, dataGraph, metric.observationURI);
              await buildResultArray(query);
			  break;
			case 'ExtensionalConcisenessMetric':
			console.log("ExtensionalConcisenessMetric");
			  query = ExtensionalConcisenessMetricQuery_FetchProblem.ExtensionalConcisenessMetric(qualityGraph, dataGraph, metric.observationURI);
			  let queryResultsTemp = await buildextensionalConcisenessMetricProblems(query);
              //console.log(queryResultsTemp);
			  for(const problem of queryResultsTemp)
			  {
				  //console.log("PROBLEM : "+problem.Subject);
				  let subject = problem.Subject;
				  query = ExtensionalConcisenessMetricQuery_FetchAllTriples.ExtensionalConcisenessMetric(qualityGraph, dataGraph, subject);
				  let triples = await buildextensionalConcisenessAllTripleQuery(query);
				  //console.log(triples);
				  let relatedResourcesQuery=ExtensionalConcisenessMetricQuery_FindRelated.ExtensionalConcisenessMetric(triples, dataGraph, subject);
				  let relatedResources=await buildextensionalConcisenessResultArray(relatedResourcesQuery);
				  //console.log(relatedResources);
				  
				  for(const related of relatedResources)
				  {
					  problem.Object=related;
					  queryResults.push(problem);
				  }
				  
			  }

			  break;
            default:
			console.log("default");
              query = "";
			  break;
          }
          
        }
      }

      triggerActivity().then(async function () {
        //Loop through each array object
        for (const [outerIndex, outerResult] of queryResults.entries()) {
          let SubjectRelatedExceptions = [];
          let PredicateRelatedExceptions = [];
          let ObjectRelatedExceptions = [];
          let SameTripleSameProblematicThingRelatedExceptions = [];
          let SameTripleDifferentProblematicThingRelatedExceptions = [];
          let tempArray = [];
          //console.log(outerIndex);
          //Add Await call to setImmediate to prevent thread blocking 
          await setImmediatePromise()
          //Find SubjectRelatedExceptions
          tempArray = [];
          for (const [innerIndex, innerResult] of queryResults.entries()) {
            //Ignore same entry
            if (innerIndex !== outerIndex) {
              if (innerResult.ProblematicThing === "Subject" & outerResult.Subject === innerResult.Subject) {
                tempArray.push(innerResult.Exception);
              }
            }
          }
          SubjectRelatedExceptions = [...new Set(tempArray.map((exception) => (exception)))];
          queryResults[outerIndex].SubjectRelatedExceptions = [...SubjectRelatedExceptions];



          //Find PredicateRelatedExceptions
          tempArray = [];
          for (const [innerIndex, innerResult] of queryResults.entries()) {
            //Ignore same entry
            if (innerIndex !== outerIndex) {
              if (innerResult.ProblematicThing === "Predicate" & outerResult.Predicate === innerResult.Predicate) {
                tempArray.push(innerResult.Exception);
              }
            }
          }
          PredicateRelatedExceptions = [...new Set(tempArray.map((exception) => (exception)))];
          queryResults[outerIndex].PredicateRelatedExceptions = [...PredicateRelatedExceptions];



          //Find ObjectRelatedExceptions
          tempArray = [];
          for (const [innerIndex, innerResult] of queryResults.entries()) {
            //Ignore same entry
            if (innerIndex !== outerIndex) {
              if (innerResult.ProblematicThing === "Object" & outerResult.Object === innerResult.Object) {
                tempArray.push(innerResult.Exception);
              }
            }
          }
          ObjectRelatedExceptions = [...new Set(tempArray.map((exception) => (exception)))];
          queryResults[outerIndex].ObjectRelatedExceptions = [...ObjectRelatedExceptions];



          //Find SameTripleSameProblematicThingRelatedExceptions
          tempArray = [];
          for (const [innerIndex, innerResult] of queryResults.entries()) {
            //Ignore same entry
            if (innerIndex !== outerIndex) {
              if (innerResult.ProblematicThing === outerResult.ProblematicThing & outerResult.Subject === innerResult.Subject & outerResult.Precicate === innerResult.Precicate & outerResult.Object === innerResult.Object) {
                tempArray.push(innerResult.Exception);
              }
            }
          }
          SameTripleSameProblematicThingRelatedExceptions = [...new Set(tempArray.map((exception) => (exception)))];
          queryResults[outerIndex].SameTripleSameProblematicThingRelatedExceptions = [...SameTripleSameProblematicThingRelatedExceptions];



          //Find SameTripleDifferentProblematicThingRelatedExceptions
          tempArray = [];
          for (const [innerIndex, innerResult] of queryResults.entries()) {
            //Ignore same entry
            if (innerIndex !== outerIndex) {
              if (innerResult.ProblematicThing !== outerResult.ProblematicThing & outerResult.Subject === innerResult.Subject & outerResult.Precicate === innerResult.Precicate & outerResult.Object === innerResult.Object) {
                tempArray.push(innerResult.Exception);
              }
            }
          }
          SameTripleDifferentProblematicThingRelatedExceptions = [...new Set(tempArray.map((exception) => (exception)))];
          queryResults[outerIndex].SameTripleDifferentProblematicThingRelatedExceptions = [...SameTripleDifferentProblematicThingRelatedExceptions];
          queryResults[outerIndex].knowledgeBaseID = knowledgeBaseID;

          //Write to MongoDB
		  //console.log(queryResults);
          let knowledgeBaseDocumentToSave = new knowledgeBaseModel(queryResults[outerIndex]);
          await knowledgeBaseDocumentToSave.save();

        }

/*
        for (let row = 0; row < queryResults.length; row++) {
          let keysAmount = Object.keys(queryResults[row]).length
          let keysCounter = 0

          // If this is the first row, generate the headings
          if (row === 0) {

            // Loop each property of the object
            for (let key in queryResults[row]) {
              output += key + (keysCounter + 1 < keysAmount ? '\t' : '\r\n')
              keysCounter++
            }
          } else {
            for (let key in queryResults[row]) {
              output += queryResults[row][key] + (keysCounter + 1 < keysAmount ? '\t' : '\r\n')
              keysCounter++
            }
          }
          keysCounter = 0
        }
*/
        //console.log(output)


        //Write to File
        /*        fs.writeFile(knowledgeBaseFile + '/formList.csv', output, 'utf8', function (err) {
                  if (err) {
                    //console.log('Some error occured - file either not saved or corrupted file saved.');
                  } else {
                    //console.log('It\'s saved!');
                  }
                });
        */
        //Update Status as Completed
        rawdata = fs.readFileSync(statusFile);
        knowledgeBase = JSON.parse(rawdata);
        let knowledgeBaseIDIndex;
        knowledgeBase["knowledgeBases"].map((knowledgeBase, index) => {
          if (knowledgeBase.knowledgeBaseID === knowledgeBaseID) {
            knowledgeBaseIDIndex = index;
          }
        });
        knowledgeBase["knowledgeBases"][knowledgeBaseIDIndex].status = "COMPLETED";
        let updatedData = JSON.stringify(knowledgeBase, null, 2);
        fs.writeFileSync(statusFile, updatedData);

      }
      ).catch(function(error){
     rawdata = fs.readFileSync(statusFile);
        knowledgeBase = JSON.parse(rawdata);
        let knowledgeBaseIDIndex;
        knowledgeBase["knowledgeBases"].map((knowledgeBase, index) => {
          if (knowledgeBase.knowledgeBaseID === knowledgeBaseID) {
            knowledgeBaseIDIndex = index;
          }
        });
        knowledgeBase["knowledgeBases"][knowledgeBaseIDIndex].status = "FAILED";
        let updatedData = JSON.stringify(knowledgeBase, null, 2);
        fs.writeFileSync(statusFile, updatedData);
	  });

      return res.status(201).send(knowledgeBaseNew);

    });


  }
}

const knowledgeBase = new knowledgeBases();
export default knowledgeBase;