Use existing scalar fields implemented

confusion matrix update

ContextBasedFeature.cpp

@@ -119,6 +119,7 @@ bool ContextBasedFeature::prepare(	const CorePoints& corePoints,
 
 	//and the scalar field
 	assert(!sf);
+	sfWasAlreadyExisting = CheckSFExistence(corePoints.cloud, qPrintable(resultSFName));
 	sf = PrepareSF(corePoints.cloud, qPrintable(resultSFName), generatedScalarFields, SFCollector::CAN_REMOVE);
 	if (!sf)
 	{
@@ -127,7 +128,8 @@ bool ContextBasedFeature::prepare(	const CorePoints& corePoints,
 	}
 	source.name = sf->getName();
 
-	if (!scaled()) //with 'kNN' neighbors, we can compute the values right away
+	// NOT NECESSARY IF THE VALUE IS ALREADY COMPUTED
+	if (!scaled() && !sfWasAlreadyExisting) //with 'kNN' neighbors, we can compute the values right away
 	{
 		unsigned pointCount = corePoints.size();
 		QString logMessage = QString("Computing %1 on cloud %2 with context cloud %3\n(core points: %4)").arg(typeStr).arg(corePoints.cloud->getName()).arg(cloud2Label).arg(pointCount);

ContextBasedFeature.h

@@ -75,6 +75,7 @@ namespace masc
 			, kNN(p_kNN)
 			, ctxClassLabel(p_ctxClassLabel)
 			, sf(nullptr)
+			, sfWasAlreadyExisting(false)
 		{
 			scale = p_scale;
 		}
@@ -103,5 +104,7 @@ namespace masc
 		int ctxClassLabel;
 		//! The computed scalar
 		CCCoreLib::ScalarField* sf;
+
+		bool sfWasAlreadyExisting;
 	};
-}
+}

FeaturesInterface.cpp

@@ -25,6 +25,19 @@
 
 using namespace masc;
 
+bool Feature::CheckSFExistence(ccPointCloud* cloud, const char* resultSFName)
+{
+	int sfIdx = cloud->getScalarFieldIndexByName(resultSFName);
+	if (sfIdx >= 0)
+	{
+		return true;
+	}
+	else
+	{
+		return false;
+	}
+}
+
 CCCoreLib::ScalarField* Feature::PrepareSF(ccPointCloud* cloud, const char* resultSFName, SFCollector* generatedScalarFields/*=nullptr*/, SFCollector::Behavior behavior/*=SFCollector::CAN_REMOVE*/)
 {
 	if (!cloud || !resultSFName)
@@ -38,10 +51,13 @@ CCCoreLib::ScalarField* Feature::PrepareSF(ccPointCloud* cloud, const char* resu
 	int sfIdx = cloud->getScalarFieldIndexByName(resultSFName);
 	if (sfIdx >= 0)
 	{
+		ccLog::Warning("Existing SF: " + QString(resultSFName) + ", SFCollector::Behavior " + QString::number(behavior));
 		resultSF = cloud->getScalarField(sfIdx);
 	}
 	else
 	{
+		ccLog::Warning("SF does not exist, create it: " + QString(resultSFName)  + ", SFCollector::Behavior " + QString::number(behavior));
+		resultSF = cloud->getScalarField(sfIdx);
 		ccScalarField* newSF = new ccScalarField(resultSFName);
 		if (!newSF->resizeSafe(cloud->size()))
 		{

FeaturesInterface.h

@@ -209,6 +209,9 @@ namespace masc
 
 	public: //helpers
 
+		//! Creates (or resets) a scalar field with the given name on the input core points cloud
+		static bool CheckSFExistence(ccPointCloud* cloud, const char* resultSFName);
+
 		//! Creates (or resets) a scalar field with the given name on the input core points cloud
 		static CCCoreLib::ScalarField* PrepareSF(ccPointCloud* cloud, const char* resultSFName, SFCollector* generatedScalarFields/*= nullptr*/, SFCollector::Behavior behavior);
 

NeighborhoodFeature.cpp

@@ -89,7 +89,11 @@ bool NeighborhoodFeature::prepare(	const CorePoints& corePoints,
 
 	//and the scalar field
 	assert(!sf1);
-	sf1 = PrepareSF(corePoints.cloud, qPrintable(resultSFName), generatedScalarFields, SFCollector::CAN_REMOVE);
+	sf1WasAlreadyExisting = CheckSFExistence(corePoints.cloud, qPrintable(resultSFName));
+	if (sf1WasAlreadyExisting)
+		sf1 = PrepareSF(corePoints.cloud, qPrintable(resultSFName), generatedScalarFields, SFCollector::ALWAYS_KEEP);
+	else
+		sf1 = PrepareSF(corePoints.cloud, qPrintable(resultSFName), generatedScalarFields, SFCollector::CAN_REMOVE);
 	if (!sf1)
 	{
 		error = QString("Failed to prepare scalar %1 @ scale %2").arg(resultSFName).arg(scale);
@@ -97,13 +101,18 @@ bool NeighborhoodFeature::prepare(	const CorePoints& corePoints,
 	}
 	source.name = sf1->getName();
 
-	if (cloud2 && op != Feature::NO_OPERATION)
+	// sf2 is not needed if sf1 was already existing!
+	if (cloud2 && op != Feature::NO_OPERATION && !sf1WasAlreadyExisting)
 	{
 		QString resultSFName2 = ToString(type) + "_" + cloud2Label + "@" + QString::number(scale);
 		keepSF2 = (corePoints.cloud->getScalarFieldIndexByName(qPrintable(resultSFName2)) >= 0); //we remember that the scalar field was already existing!
 
 		assert(!sf2);
-		sf2 = PrepareSF(corePoints.cloud, qPrintable(resultSFName2), generatedScalarFields, SFCollector::ALWAYS_REMOVE);
+		sf2WasAlreadyExisting = CheckSFExistence(corePoints.cloud, qPrintable(resultSFName2));
+		if (sf2WasAlreadyExisting)
+			sf2 = PrepareSF(corePoints.cloud, qPrintable(resultSFName2), generatedScalarFields, SFCollector::ALWAYS_KEEP);
+		else
+			sf2 = PrepareSF(corePoints.cloud, qPrintable(resultSFName2), generatedScalarFields, SFCollector::ALWAYS_REMOVE);
 		if (!sf2)
 		{
 			error = QString("Failed to prepare scalar field for %1 @ scale %2").arg(cloud2Label).arg(scale);
@@ -140,7 +149,7 @@ bool NeighborhoodFeature::finish(const CorePoints& corePoints, QString& error)
 		}
 	}
 
-	if (sf2)
+	if (sf2 && !sf1WasAlreadyExisting)
 	{
 		//now perform the math operation
 		if (op != Feature::NO_OPERATION)
@@ -152,24 +161,24 @@ bool NeighborhoodFeature::finish(const CorePoints& corePoints, QString& error)
 			}
 		}
 
-		if (keepSF2)
-		{
-			sf2->computeMinAndMax();
-		}
-		else
-		{
-			int sfIndex2 = corePoints.cloud->getScalarFieldIndexByName(sf2->getName());
-			if (sfIndex2 >= 0)
-			{
-				corePoints.cloud->deleteScalarField(sfIndex2);
-			}
-			else
-			{
-				assert(false);
-				sf2->release();
-			}
-			sf2 = nullptr;
-		}
+//		if (keepSF2)
+//		{
+//			sf2->computeMinAndMax();
+//		}
+//		else
+//		{
+//			int sfIndex2 = corePoints.cloud->getScalarFieldIndexByName(sf2->getName());
+//			if (sfIndex2 >= 0)
+//			{
+//				corePoints.cloud->deleteScalarField(sfIndex2);
+//			}
+//			else
+//			{
+//				assert(false);
+//				sf2->release();
+//			}
+//			sf2 = nullptr;
+//		}
 	}
 
 	return success;

NeighborhoodFeature.h

@@ -152,6 +152,8 @@ namespace masc
 			, sf1(nullptr)
 			, sf2(nullptr)
 			, keepSF2(false)
+			, sf1WasAlreadyExisting(false)
+			, sf2WasAlreadyExisting(false)
 		{
 		}
 
@@ -176,5 +178,7 @@ namespace masc
 		//! Feature values
 		CCCoreLib::ScalarField *sf1, *sf2;
 		bool keepSF2;
+		bool sf1WasAlreadyExisting;
+		bool sf2WasAlreadyExisting;
 	};
-}
+}

PointFeature.cpp

@@ -577,7 +577,11 @@ bool PointFeature::prepare(	const CorePoints& corePoints,
 		resultSFName += "@" + QString::number(scale);
 
 		//prepare the corresponding scalar field
-		statSF1 = PrepareSF(corePoints.cloud, qPrintable(resultSFName), generatedScalarFields, SFCollector::CAN_REMOVE);
+		statSF1WasAlreadyExisting = CheckSFExistence(corePoints.cloud, qPrintable(resultSFName));
+		if (statSF1WasAlreadyExisting)
+			statSF1 = PrepareSF(corePoints.cloud, qPrintable(resultSFName), generatedScalarFields, SFCollector::ALWAYS_KEEP);
+		else
+			statSF1 = PrepareSF(corePoints.cloud, qPrintable(resultSFName), generatedScalarFields, SFCollector::CAN_REMOVE);
 		if (!statSF1)
 		{
 			error = QString("Failed to prepare scalar field for field '%1' @ scale %2").arg(field1->getName()).arg(scale);
@@ -585,20 +589,24 @@ bool PointFeature::prepare(	const CorePoints& corePoints,
 		}
 		source.name = statSF1->getName();
 
-		if (field2 && op != Feature::NO_OPERATION)
+		if (field2 && op != Feature::NO_OPERATION && !statSF1WasAlreadyExisting) // nothing to do if statSF1 was already there
 		{
 			QString resultSFName2 = field2->getName() + QString("_") + Feature::StatToString(stat) + "_" + cloud2Label + "@" + QString::number(scale);
 			//keepStatSF2 = (corePoints.cloud->getScalarFieldIndexByName(qPrintable(resultSFName2)) >= 0); //we remember that the scalar field was already existing!
 
 			assert(!statSF2);
-			statSF2 = PrepareSF(corePoints.cloud, qPrintable(resultSFName2), generatedScalarFields, SFCollector::ALWAYS_REMOVE);
+			statSF2WasAlreadyExisting = CheckSFExistence(corePoints.cloud, qPrintable(resultSFName2));
+			if (statSF2WasAlreadyExisting)
+				statSF2 = PrepareSF(corePoints.cloud, qPrintable(resultSFName2), generatedScalarFields, SFCollector::ALWAYS_KEEP);
+			else
+				statSF2 = PrepareSF(corePoints.cloud, qPrintable(resultSFName2), generatedScalarFields, SFCollector::ALWAYS_REMOVE);
 			if (!statSF2)
 			{
 				error = QString("Failed to prepare scalar field for field '%1' @ scale %2").arg(field2->getName()).arg(scale);
 				return false;
 			}
 		}
-		
+
 		return true;
 	}
 	else //non scaled feature
@@ -843,7 +851,7 @@ bool PointFeature::finish(const CorePoints& corePoints, QString& error)
 		}
 	}
 
-	if (statSF2)
+	if (statSF2 && !statSF1WasAlreadyExisting)
 	{
 		//now perform the math operation
 		if (op != Feature::NO_OPERATION)
@@ -854,7 +862,7 @@ bool PointFeature::finish(const CorePoints& corePoints, QString& error)
 				success = false;
 			}
 		}
-		statSF2->computeMinAndMax();
+//		statSF2->computeMinAndMax();
 
 		//DGM: we don't delete it now! As it could be used by other features!
 		//if (!keepStatSF2)

PointFeature.h

@@ -148,6 +148,8 @@ namespace masc
 			, field2(nullptr)
 			, statSF1(nullptr)
 			, statSF2(nullptr)
+			, statSF1WasAlreadyExisting(false)
+			, statSF2WasAlreadyExisting(false)
 			//, keepStatSF2(false)
 		{
 			//auomatically set the right source for specific features
@@ -213,5 +215,7 @@ namespace masc
 		CCCoreLib::ScalarField *statSF1, *statSF2;
 		
 		//bool keepStatSF2;
+		bool statSF1WasAlreadyExisting;
+		bool statSF2WasAlreadyExisting;
 	};
 }

ScalarFieldCollector.cpp

@@ -50,6 +50,7 @@ void SFCollector::releaseSFs(bool keepByDefault)
 		int sfIdx = desc.cloud->getScalarFieldIndexByName(sf->getName());
 		if (sfIdx >= 0)
 		{
+			ccLog::Warning(QString("[SFCollector] Remove scalar field '%1'").arg(sf->getName()));
 			desc.cloud->deleteScalarField(sfIdx);
 		}
 		else

confusionmatrix.cpp

@@ -9,6 +9,20 @@
 
 #include <QBrush>
 
+#include <ccLog.h>
+
+QColor getColor(double value, double r1, double g1, double b1)
+{
+	double r0 = 255;
+	double g0 = 255;
+	double b0 = 255;
+	int r = int((r1 - r0) * value + r0);
+	int g = int ((g1 - g0) * value + g0);
+	int b = int ((b1 - b0) * value + b0);
+	ccLog::Warning("value " + QString::number(value) + " (" + QString::number(r) + ", " + QString::number(g) + ", " + QString::number(b) + ")");
+	return QColor(r, g, b);
+}
+
 ConfusionMatrix::ConfusionMatrix(std::vector<ScalarType> &actual, std::vector<ScalarType> &predicted, QWidget *parent) :
 	QWidget(parent),
 	ui(new Ui::ConfusionMatrix)
@@ -17,7 +31,12 @@ ConfusionMatrix::ConfusionMatrix(std::vector<ScalarType> &actual, std::vector<Sc
 	this->setWindowFlag(Qt::WindowStaysOnTopHint);
 	compute(actual, predicted);
 	this->show();
-	this->setMinimumSize(this->ui->tableWidget->sizeHint());
+	this->ui->tableWidget->resizeColumnsToContents();
+	this->ui->tableWidget->setSizeAdjustPolicy(QAbstractScrollArea::AdjustToContents);
+	QSize tableSize = this->ui->tableWidget->sizeHint();
+	QSize labelSize = this->ui->label->sizeHint();
+	QSize widgetSize = QSize(tableSize.width(), tableSize.height() + labelSize.height());
+	this->setMinimumSize(widgetSize);
 }
 
 ConfusionMatrix::~ConfusionMatrix()
@@ -25,7 +44,7 @@ ConfusionMatrix::~ConfusionMatrix()
 	delete ui;
 }
 
-void ConfusionMatrix::computePrecisionRecallF1Score(cv::Mat& matrix, cv::Mat& precisionRecallF1Score)
+void ConfusionMatrix::computePrecisionRecallF1Score(cv::Mat& matrix, cv::Mat& precisionRecallF1Score, cv::Mat& vec_TP_FN)
 {
 	int nbClasses = matrix.rows;
 
@@ -65,6 +84,7 @@ void ConfusionMatrix::computePrecisionRecallF1Score(cv::Mat& matrix, cv::Mat& pr
 			precisionRecallF1Score.at<float>(realIdx, RECALL) = CCCoreLib::NAN_VALUE;
 		else
 			precisionRecallF1Score.at<float>(realIdx, RECALL) = TP / TP_FN;
+		vec_TP_FN.at<int>(realIdx, 0) = TP_FN;
 	}
 
 	// compute F1-score
@@ -122,6 +142,7 @@ void ConfusionMatrix::compute(std::vector<ScalarType>& actual, std::vector<Scala
 	int nbClasses = classes.size();
 	cv::Mat confusionMatrix(nbClasses, nbClasses, CV_32S, cv::Scalar(0));
 	cv::Mat precisionRecallF1Score(nbClasses, 3, CV_32F, cv::Scalar(0));
+	cv::Mat vec_TP_FN(nbClasses, 1, CV_32S, cv::Scalar(0));
 
 	// fill the confusion matrix
 	for (int i = 0; i < actual.size(); i++)
@@ -134,7 +155,7 @@ void ConfusionMatrix::compute(std::vector<ScalarType>& actual, std::vector<Scala
 	}
 
 	// compute precision recall F1-score
-	computePrecisionRecallF1Score(confusionMatrix, precisionRecallF1Score);
+	computePrecisionRecallF1Score(confusionMatrix, precisionRecallF1Score, vec_TP_FN);
 	float overallAccuracy = computeOverallAccuracy(confusionMatrix);
 
 	// display the overall accuracy
@@ -164,8 +185,8 @@ void ConfusionMatrix::compute(std::vector<ScalarType>& actual, std::vector<Scala
 	newItem->setBackground(Qt::lightGray);
 	newItem->setTextAlignment(Qt::AlignCenter);
 	this->ui->tableWidget->setItem(0, 2, newItem);
-	// Actual
-	newItem = new QTableWidgetItem("Actual");
+	// Real
+	newItem = new QTableWidgetItem("Real");
 	newItem->setFont(font);
 	newItem->setBackground(Qt::lightGray);
 	newItem->setTextAlignment(Qt::AlignCenter);
@@ -202,11 +223,12 @@ void ConfusionMatrix::compute(std::vector<ScalarType>& actual, std::vector<Scala
 	for (int row = 0; row < nbClasses; row++)
 		for (int column = 0; column < nbClasses; column++)
 		{
-			QTableWidgetItem *newItem = new QTableWidgetItem(QString::number(confusionMatrix.at<int>(row, column)));
+			double val = confusionMatrix.at<int>(row, column);
+			QTableWidgetItem *newItem = new QTableWidgetItem(QString::number(val));
 			if (row == column)
-				newItem->setBackground(greenBrush); // green QColor(37, 190, 147, 1)
+				newItem->setBackground(getColor(val / vec_TP_FN.at<int>(row, 0), 0, 128, 255));
 			else
-				newItem->setBackground(QColorConstants::Svg::orange); // QColor(255, 129, 129, 1)
+				newItem->setBackground(getColor(val / vec_TP_FN.at<int>(row, 0), 0, 128, 255));
 			this->ui->tableWidget->setItem(2 + row, + 2 + column, newItem);
 		}
 

confusionmatrix.h

@@ -26,7 +26,7 @@ public:
 	explicit ConfusionMatrix(std::vector<ScalarType>& actual, std::vector<ScalarType>& predicted, QWidget *parent = nullptr);
 	~ConfusionMatrix();
 
-	void computePrecisionRecallF1Score(cv::Mat& matrix, cv::Mat& precisionRecallF1Score);
+	void computePrecisionRecallF1Score(cv::Mat& matrix, cv::Mat& precisionRecallF1Score, cv::Mat &vec_TP_FN);
 	float computeOverallAccuracy(cv::Mat& matrix);
 	void compute(std::vector<ScalarType>& actual, std::vector<ScalarType>& predicted);
 	void setSessionRun(QString session, int run);

q3DMASCTools.cpp

@@ -178,6 +178,19 @@ bool Tools::LoadClassifierCloudLabels(QString filename, QList<QString>& labels,
 	return true;
 }
 
+bool CheckFeatureUnicity(std::vector<Feature::Shared>& rawFeatures, Feature::Shared feature)
+{
+	// check that the feature does not exists already!
+	for (const auto &feat : rawFeatures)
+	{
+		if (feat->toString() == feature->toString())
+		{
+			return false;
+		}
+	}
+	return true;
+}
+
 static bool CreateFeaturesFromCommand(const QString& command, QString corePointsRole, int lineNumber, const Tools::NamedClouds& clouds, std::vector<Feature::Shared>& rawFeatures, std::vector<double>& scales)
 {
 	QStringList tokens = command.split('_');
@@ -470,8 +483,16 @@ static bool CreateFeaturesFromCommand(const QString& command, QString corePoints
 		return false;
 	}
 
-	//save it
-	rawFeatures.push_back(feature);
+	if (!CheckFeatureUnicity(rawFeatures, feature)) // check that the feature does not exists already!
+	{
+		ccLog::Warning("[3DMASC] duplicated feature " + feature->toString() + ", check your parameter file");
+		return false;
+	}
+	else
+	{
+		//save the feature
+		rawFeatures.push_back(feature);
+	}
 
 	if (useAllScales)
 	{
@@ -484,7 +505,16 @@ static bool CreateFeaturesFromCommand(const QString& command, QString corePoints
 			//as we only change the scale value, all the duplicated features should be valid
 			assert(newFeature->checkValidity(corePointsRole, errorMessage));
 
-			rawFeatures.push_back(newFeature);
+			if (!CheckFeatureUnicity(rawFeatures, newFeature)) // check that the feature does not exists already!
+			{
+				ccLog::Warning("[3DMASC] duplicated feature " + newFeature->toString() + ", check your parameter file");
+				return false;
+			}
+			else
+			{
+				//save the feature
+				rawFeatures.push_back(newFeature);
+			}
 		}
 	}
 
@@ -1001,7 +1031,8 @@ bool Tools::PrepareFeatures(const CorePoints& corePoints, Feature::Set& features
 				case Feature::Type::PointFeature:
 				{
 					//build the scaled feature list attached to the first cloud
-					if (feature->cloud1)
+					if (feature->cloud1
+						&& !static_cast<PointFeature*>(feature.data())->statSF1WasAlreadyExisting) // nothing to compute if the scalar field was already there
 					{
 						FeaturesAndScales& fas = cloudsWithScaledFeatures[feature->cloud1];
 						fas.pointFeaturesPerScale[feature->scale].push_back(qSharedPointerCast<PointFeature>(feature));
@@ -1013,7 +1044,8 @@ bool Tools::PrepareFeatures(const CorePoints& corePoints, Feature::Set& features
 					}
 
 					//build the scaled feature list attached to the second cloud (if any)
-					if (feature->cloud2 && feature->cloud2 != feature->cloud1 && feature->op != Feature::NO_OPERATION)
+					if (feature->cloud2 && feature->cloud2 != feature->cloud1 && feature->op != Feature::NO_OPERATION
+						&& !static_cast<PointFeature*>(feature.data())->statSF1WasAlreadyExisting) // nothing to compute if the scalar field was already there
 					{
 						FeaturesAndScales& fas = cloudsWithScaledFeatures[feature->cloud2];
 						++fas.featureCount;
@@ -1030,7 +1062,8 @@ bool Tools::PrepareFeatures(const CorePoints& corePoints, Feature::Set& features
 				case Feature::Type::NeighborhoodFeature:
 				{
 					//build the scaled feature list attached to the first cloud
-					if (feature->cloud1)
+					if (feature->cloud1
+						&& !static_cast<NeighborhoodFeature*>(feature.data())->sf1WasAlreadyExisting) // nothing to compute if the scalar field was already there
 					{
 						FeaturesAndScales& fas = cloudsWithScaledFeatures[feature->cloud1];
 						fas.neighborhoodFeaturesPerScale[feature->scale].push_back(qSharedPointerCast<NeighborhoodFeature>(feature));
@@ -1042,7 +1075,8 @@ bool Tools::PrepareFeatures(const CorePoints& corePoints, Feature::Set& features
 					}
 
 					//build the scaled feature list attached to the second cloud (if any)
-					if (feature->cloud2 && feature->cloud2 != feature->cloud1 && feature->op != Feature::NO_OPERATION)
+					if (feature->cloud2 && feature->cloud2 != feature->cloud1 && feature->op != Feature::NO_OPERATION
+						&& !static_cast<NeighborhoodFeature*>(feature.data())->sf1WasAlreadyExisting) // nothing to compute if the scalar field was already there
 					{
 						FeaturesAndScales& fas = cloudsWithScaledFeatures[feature->cloud2];
 						fas.neighborhoodFeaturesPerScale[feature->scale].push_back(qSharedPointerCast<NeighborhoodFeature>(feature));
@@ -1059,7 +1093,8 @@ bool Tools::PrepareFeatures(const CorePoints& corePoints, Feature::Set& features
 				case Feature::Type::ContextBasedFeature:
 				{
 					//build the scaled feature list attached to the second cloud (the 'context' cloud)
-					if (feature->cloud2)
+					if (feature->cloud2
+						&& !static_cast<ContextBasedFeature*>(feature.data())->sfWasAlreadyExisting) // nothing to compute if the scalar field was already there
 					{
 						FeaturesAndScales& fas = cloudsWithScaledFeatures[feature->cloud2];
 						fas.contextBasedFeaturesPerScale[feature->scale].push_back(qSharedPointerCast<ContextBasedFeature>(feature));