diff --git a/Test/Automated/Fixture/Comparison/RecipesA/rtbMakeCoordinatesTest/renderings/Mitsuba/scene-001.mat b/Test/Automated/Fixture/Comparison/RecipesA/rtbMakeCoordinatesTest/renderings/Mitsuba/scene-001.mat
new file mode 100644
index 0000000000000000000000000000000000000000..91b97c1fb5bd7b3f3f3430d01789d7853c2c57e1
Binary files /dev/null and b/Test/Automated/Fixture/Comparison/RecipesA/rtbMakeCoordinatesTest/renderings/Mitsuba/scene-001.mat differ
diff --git a/Test/Automated/Fixture/Comparison/RecipesA/rtbMakeCoordinatesTest/renderings/PBRT/scene-001.mat b/Test/Automated/Fixture/Comparison/RecipesA/rtbMakeCoordinatesTest/renderings/PBRT/scene-001.mat
new file mode 100644
index 0000000000000000000000000000000000000000..87847b68e8bb9a9c672bf3f862a1f794973df60a
Binary files /dev/null and b/Test/Automated/Fixture/Comparison/RecipesA/rtbMakeCoordinatesTest/renderings/PBRT/scene-001.mat differ
diff --git a/Test/Automated/Fixture/Comparison/RecipesA/rtbMakeMaterialSphereBumps/renderings/Mitsuba/materialSphereMatte.mat b/Test/Automated/Fixture/Comparison/RecipesA/rtbMakeMaterialSphereBumps/renderings/Mitsuba/materialSphereMatte.mat
new file mode 100644
index 0000000000000000000000000000000000000000..147757056e2f6d9761765ad2b533fe88fa5fa65a
Binary files /dev/null and b/Test/Automated/Fixture/Comparison/RecipesA/rtbMakeMaterialSphereBumps/renderings/Mitsuba/materialSphereMatte.mat differ
diff --git a/Test/Automated/Fixture/Comparison/RecipesA/rtbMakeMaterialSphereBumps/renderings/Mitsuba/materialSphereMetal.mat b/Test/Automated/Fixture/Comparison/RecipesA/rtbMakeMaterialSphereBumps/renderings/Mitsuba/materialSphereMetal.mat
new file mode 100644
index 0000000000000000000000000000000000000000..ec74afcd8290b20e9f36097cba384fe64cb5c3d8
Binary files /dev/null and b/Test/Automated/Fixture/Comparison/RecipesA/rtbMakeMaterialSphereBumps/renderings/Mitsuba/materialSphereMetal.mat differ
diff --git a/Test/Automated/Fixture/Comparison/RecipesA/rtbMakeMaterialSphereBumps/renderings/Mitsuba/materialSphereWard.mat b/Test/Automated/Fixture/Comparison/RecipesA/rtbMakeMaterialSphereBumps/renderings/Mitsuba/materialSphereWard.mat
new file mode 100644
index 0000000000000000000000000000000000000000..ab258e33129912f2dad8db80ce287b2d46accd4b
Binary files /dev/null and b/Test/Automated/Fixture/Comparison/RecipesA/rtbMakeMaterialSphereBumps/renderings/Mitsuba/materialSphereWard.mat differ
diff --git a/Test/Automated/Fixture/Comparison/RecipesA/rtbMakeMaterialSphereBumps/renderings/PBRT/materialSphereMatte.mat b/Test/Automated/Fixture/Comparison/RecipesA/rtbMakeMaterialSphereBumps/renderings/PBRT/materialSphereMatte.mat
new file mode 100644
index 0000000000000000000000000000000000000000..ae0d5c3171187ac6cf9f51b15d0831f13912d524
Binary files /dev/null and b/Test/Automated/Fixture/Comparison/RecipesA/rtbMakeMaterialSphereBumps/renderings/PBRT/materialSphereMatte.mat differ
diff --git a/Test/Automated/Fixture/Comparison/RecipesA/rtbMakeMaterialSphereBumps/renderings/PBRT/materialSphereMetal.mat b/Test/Automated/Fixture/Comparison/RecipesA/rtbMakeMaterialSphereBumps/renderings/PBRT/materialSphereMetal.mat
new file mode 100644
index 0000000000000000000000000000000000000000..581b93e5b347f15f1ba7d0cdecaaea7ec0cc5a3d
Binary files /dev/null and b/Test/Automated/Fixture/Comparison/RecipesA/rtbMakeMaterialSphereBumps/renderings/PBRT/materialSphereMetal.mat differ
diff --git a/Test/Automated/Fixture/Comparison/RecipesA/rtbMakeMaterialSphereBumps/renderings/PBRT/materialSphereWard.mat b/Test/Automated/Fixture/Comparison/RecipesA/rtbMakeMaterialSphereBumps/renderings/PBRT/materialSphereWard.mat
new file mode 100644
index 0000000000000000000000000000000000000000..531b93f5db0a6607ae6846f8c1ee2beee5dc0aff
Binary files /dev/null and b/Test/Automated/Fixture/Comparison/RecipesA/rtbMakeMaterialSphereBumps/renderings/PBRT/materialSphereWard.mat differ
diff --git a/Test/Automated/Fixture/Comparison/RecipesA/rtbMakeMaterialSphereRemodeled/renderings/Mitsuba/materialSphereMatte.mat b/Test/Automated/Fixture/Comparison/RecipesA/rtbMakeMaterialSphereRemodeled/renderings/Mitsuba/materialSphereMatte.mat
new file mode 100644
index 0000000000000000000000000000000000000000..9581989dddc44c36f608e625b8f8d5526b23cebe
Binary files /dev/null and b/Test/Automated/Fixture/Comparison/RecipesA/rtbMakeMaterialSphereRemodeled/renderings/Mitsuba/materialSphereMatte.mat differ
diff --git a/Test/Automated/Fixture/Comparison/RecipesA/rtbMakeMaterialSphereRemodeled/renderings/Mitsuba/materialSphereMetal.mat b/Test/Automated/Fixture/Comparison/RecipesA/rtbMakeMaterialSphereRemodeled/renderings/Mitsuba/materialSphereMetal.mat
new file mode 100644
index 0000000000000000000000000000000000000000..68288e28e102fc195ba0b794eae61485c586d794
Binary files /dev/null and b/Test/Automated/Fixture/Comparison/RecipesA/rtbMakeMaterialSphereRemodeled/renderings/Mitsuba/materialSphereMetal.mat differ
diff --git a/Test/Automated/Fixture/Comparison/RecipesA/rtbMakeMaterialSphereRemodeled/renderings/Mitsuba/materialSphereWard.mat b/Test/Automated/Fixture/Comparison/RecipesA/rtbMakeMaterialSphereRemodeled/renderings/Mitsuba/materialSphereWard.mat
new file mode 100644
index 0000000000000000000000000000000000000000..de843d723570e18cb2e736e1044026f0fd93a5a0
Binary files /dev/null and b/Test/Automated/Fixture/Comparison/RecipesA/rtbMakeMaterialSphereRemodeled/renderings/Mitsuba/materialSphereWard.mat differ
diff --git a/Test/Automated/Fixture/Comparison/RecipesA/rtbMakeMaterialSphereRemodeled/renderings/PBRT/materialSphereMatte.mat b/Test/Automated/Fixture/Comparison/RecipesA/rtbMakeMaterialSphereRemodeled/renderings/PBRT/materialSphereMatte.mat
new file mode 100644
index 0000000000000000000000000000000000000000..db91b26ce2d86e0a604c259679c02dc950380ee6
Binary files /dev/null and b/Test/Automated/Fixture/Comparison/RecipesA/rtbMakeMaterialSphereRemodeled/renderings/PBRT/materialSphereMatte.mat differ
diff --git a/Test/Automated/Fixture/Comparison/RecipesA/rtbMakeMaterialSphereRemodeled/renderings/PBRT/materialSphereMetal.mat b/Test/Automated/Fixture/Comparison/RecipesA/rtbMakeMaterialSphereRemodeled/renderings/PBRT/materialSphereMetal.mat
new file mode 100644
index 0000000000000000000000000000000000000000..141ea0c82a44a3f8213304a0a8d7aa24bac68324
Binary files /dev/null and b/Test/Automated/Fixture/Comparison/RecipesA/rtbMakeMaterialSphereRemodeled/renderings/PBRT/materialSphereMetal.mat differ
diff --git a/Test/Automated/Fixture/Comparison/RecipesA/rtbMakeMaterialSphereRemodeled/renderings/PBRT/materialSphereWard.mat b/Test/Automated/Fixture/Comparison/RecipesA/rtbMakeMaterialSphereRemodeled/renderings/PBRT/materialSphereWard.mat
new file mode 100644
index 0000000000000000000000000000000000000000..23ffa17b5bac30b8dcf5c4c2d18e1d66ebce4afb
Binary files /dev/null and b/Test/Automated/Fixture/Comparison/RecipesA/rtbMakeMaterialSphereRemodeled/renderings/PBRT/materialSphereWard.mat differ
diff --git a/Test/Automated/Fixture/Comparison/RecipesB/rtbMakeCoordinatesTest/renderings/Mitsuba/scene-001.mat b/Test/Automated/Fixture/Comparison/RecipesB/rtbMakeCoordinatesTest/renderings/Mitsuba/scene-001.mat
new file mode 100644
index 0000000000000000000000000000000000000000..ed07771aae9652a555818dfe65c126dd304bffb9
Binary files /dev/null and b/Test/Automated/Fixture/Comparison/RecipesB/rtbMakeCoordinatesTest/renderings/Mitsuba/scene-001.mat differ
diff --git a/Test/Automated/Fixture/Comparison/RecipesB/rtbMakeCoordinatesTest/renderings/PBRT/scene-001.mat b/Test/Automated/Fixture/Comparison/RecipesB/rtbMakeCoordinatesTest/renderings/PBRT/scene-001.mat
new file mode 100644
index 0000000000000000000000000000000000000000..b1d7f2875e428ab210b346360b25c78e8f62c9de
Binary files /dev/null and b/Test/Automated/Fixture/Comparison/RecipesB/rtbMakeCoordinatesTest/renderings/PBRT/scene-001.mat differ
diff --git a/Test/Automated/Fixture/Comparison/RecipesB/rtbMakeMaterialSphereBumps/renderings/Mitsuba/materialSphereMatte.mat b/Test/Automated/Fixture/Comparison/RecipesB/rtbMakeMaterialSphereBumps/renderings/Mitsuba/materialSphereMatte.mat
new file mode 100644
index 0000000000000000000000000000000000000000..a5c6d16e3fb02ff1678e85120260cac738e1c849
Binary files /dev/null and b/Test/Automated/Fixture/Comparison/RecipesB/rtbMakeMaterialSphereBumps/renderings/Mitsuba/materialSphereMatte.mat differ
diff --git a/Test/Automated/Fixture/Comparison/RecipesB/rtbMakeMaterialSphereBumps/renderings/Mitsuba/materialSphereMetal.mat b/Test/Automated/Fixture/Comparison/RecipesB/rtbMakeMaterialSphereBumps/renderings/Mitsuba/materialSphereMetal.mat
new file mode 100644
index 0000000000000000000000000000000000000000..2c66232fd55136020ebebc16e966f274763d252d
Binary files /dev/null and b/Test/Automated/Fixture/Comparison/RecipesB/rtbMakeMaterialSphereBumps/renderings/Mitsuba/materialSphereMetal.mat differ
diff --git a/Test/Automated/Fixture/Comparison/RecipesB/rtbMakeMaterialSphereBumps/renderings/Mitsuba/materialSphereWard.mat b/Test/Automated/Fixture/Comparison/RecipesB/rtbMakeMaterialSphereBumps/renderings/Mitsuba/materialSphereWard.mat
new file mode 100644
index 0000000000000000000000000000000000000000..105f862a6f126f030fd43affa8dcc1bc11c33257
Binary files /dev/null and b/Test/Automated/Fixture/Comparison/RecipesB/rtbMakeMaterialSphereBumps/renderings/Mitsuba/materialSphereWard.mat differ
diff --git a/Test/Automated/Fixture/Comparison/RecipesB/rtbMakeMaterialSphereBumps/renderings/PBRT/materialSphereMatte.mat b/Test/Automated/Fixture/Comparison/RecipesB/rtbMakeMaterialSphereBumps/renderings/PBRT/materialSphereMatte.mat
new file mode 100644
index 0000000000000000000000000000000000000000..cc408994ce9c36cf910823a70ad931fbfe4f76d2
Binary files /dev/null and b/Test/Automated/Fixture/Comparison/RecipesB/rtbMakeMaterialSphereBumps/renderings/PBRT/materialSphereMatte.mat differ
diff --git a/Test/Automated/Fixture/Comparison/RecipesB/rtbMakeMaterialSphereBumps/renderings/PBRT/materialSphereMetal.mat b/Test/Automated/Fixture/Comparison/RecipesB/rtbMakeMaterialSphereBumps/renderings/PBRT/materialSphereMetal.mat
new file mode 100644
index 0000000000000000000000000000000000000000..38975361d6d9e56af6c9b38359ed3b0e90823f87
Binary files /dev/null and b/Test/Automated/Fixture/Comparison/RecipesB/rtbMakeMaterialSphereBumps/renderings/PBRT/materialSphereMetal.mat differ
diff --git a/Test/Automated/Fixture/Comparison/RecipesB/rtbMakeMaterialSphereBumps/renderings/PBRT/materialSphereWard.mat b/Test/Automated/Fixture/Comparison/RecipesB/rtbMakeMaterialSphereBumps/renderings/PBRT/materialSphereWard.mat
new file mode 100644
index 0000000000000000000000000000000000000000..9ab6a5cca1fca86ea126f3385918601ddc0aec25
Binary files /dev/null and b/Test/Automated/Fixture/Comparison/RecipesB/rtbMakeMaterialSphereBumps/renderings/PBRT/materialSphereWard.mat differ
diff --git a/Test/Automated/Fixture/Comparison/RecipesB/rtbMakeMaterialSphereRemodeled/renderings/Mitsuba/materialSphereMatte.mat b/Test/Automated/Fixture/Comparison/RecipesB/rtbMakeMaterialSphereRemodeled/renderings/Mitsuba/materialSphereMatte.mat
new file mode 100644
index 0000000000000000000000000000000000000000..acbf171682d78a79f713be6925cd5f03054c3c08
Binary files /dev/null and b/Test/Automated/Fixture/Comparison/RecipesB/rtbMakeMaterialSphereRemodeled/renderings/Mitsuba/materialSphereMatte.mat differ
diff --git a/Test/Automated/Fixture/Comparison/RecipesB/rtbMakeMaterialSphereRemodeled/renderings/Mitsuba/materialSphereMetal.mat b/Test/Automated/Fixture/Comparison/RecipesB/rtbMakeMaterialSphereRemodeled/renderings/Mitsuba/materialSphereMetal.mat
new file mode 100644
index 0000000000000000000000000000000000000000..840feacae1e34bf9beca74a98d148bad6b04cf91
Binary files /dev/null and b/Test/Automated/Fixture/Comparison/RecipesB/rtbMakeMaterialSphereRemodeled/renderings/Mitsuba/materialSphereMetal.mat differ
diff --git a/Test/Automated/Fixture/Comparison/RecipesB/rtbMakeMaterialSphereRemodeled/renderings/Mitsuba/materialSphereWard.mat b/Test/Automated/Fixture/Comparison/RecipesB/rtbMakeMaterialSphereRemodeled/renderings/Mitsuba/materialSphereWard.mat
new file mode 100644
index 0000000000000000000000000000000000000000..fe4e6ddd9e0b008a921695f01018e4eb70f90e27
Binary files /dev/null and b/Test/Automated/Fixture/Comparison/RecipesB/rtbMakeMaterialSphereRemodeled/renderings/Mitsuba/materialSphereWard.mat differ
diff --git a/Test/Automated/Fixture/Comparison/RecipesB/rtbMakeMaterialSphereRemodeled/renderings/PBRT/materialSphereMatte.mat b/Test/Automated/Fixture/Comparison/RecipesB/rtbMakeMaterialSphereRemodeled/renderings/PBRT/materialSphereMatte.mat
new file mode 100644
index 0000000000000000000000000000000000000000..066a22de2dc7bbf77a9559701a65ef4b03d8b956
Binary files /dev/null and b/Test/Automated/Fixture/Comparison/RecipesB/rtbMakeMaterialSphereRemodeled/renderings/PBRT/materialSphereMatte.mat differ
diff --git a/Test/Automated/Fixture/Comparison/RecipesB/rtbMakeMaterialSphereRemodeled/renderings/PBRT/materialSphereMetal.mat b/Test/Automated/Fixture/Comparison/RecipesB/rtbMakeMaterialSphereRemodeled/renderings/PBRT/materialSphereMetal.mat
new file mode 100644
index 0000000000000000000000000000000000000000..caca4383f23a79a1f9129a5b2dfd23034a1c8216
Binary files /dev/null and b/Test/Automated/Fixture/Comparison/RecipesB/rtbMakeMaterialSphereRemodeled/renderings/PBRT/materialSphereMetal.mat differ
diff --git a/Test/Automated/Fixture/Comparison/RecipesB/rtbMakeMaterialSphereRemodeled/renderings/PBRT/materialSphereWard.mat b/Test/Automated/Fixture/Comparison/RecipesB/rtbMakeMaterialSphereRemodeled/renderings/PBRT/materialSphereWard.mat
new file mode 100644
index 0000000000000000000000000000000000000000..ca18a50d8c7797877026ba535cd31e457364a78e
Binary files /dev/null and b/Test/Automated/Fixture/Comparison/RecipesB/rtbMakeMaterialSphereRemodeled/renderings/PBRT/materialSphereWard.mat differ
diff --git a/Test/Automated/RtbComparisonTests.m b/Test/Automated/RtbComparisonTests.m
new file mode 100644
index 0000000000000000000000000000000000000000..c15c8d7f5a4e99b0f97d0288eb30307d311ede12
--- /dev/null
+++ b/Test/Automated/RtbComparisonTests.m
@@ -0,0 +1,174 @@
+classdef RtbComparisonTests < matlab.unittest.TestCase
+ % Test functions that find and compare renderings.
+
+ properties
+ folderA = fullfile(rtbRoot(), 'Test', 'Automated', 'Fixture', 'Comparison', 'RecipesA');
+ folderB = fullfile(rtbRoot(), 'Test', 'Automated', 'Fixture', 'Comparison', 'RecipesB');
+ scratchFolder = fullfile(tempdir(), 'RtbComparisonTests');
+ end
+
+ methods (TestMethodSetup)
+ function cleanUpOutput(testCase)
+ if 7 == exist(testCase.scratchFolder, 'dir')
+ rmdir(testCase.scratchFolder, 's');
+ end
+ end
+ end
+
+
+ methods (Test)
+
+ function testFindRenderingsSuccess(testCase)
+ expectedNames = {'rtbMakeCoordinatesTest', ...
+ 'rtbMakeMaterialSphereBumps', ...
+ 'rtbMakeMaterialSphereRemodeled'};
+
+ renderingsA = rtbFindRenderings(testCase.folderA);
+ recipeNamesA = unique({renderingsA.recipeName});
+ testCase.assertEqual(recipeNamesA, expectedNames);
+ testCase.assertNumElements(renderingsA, 14);
+
+ renderingsB = rtbFindRenderings(testCase.folderB);
+ recipeNamesB = unique({renderingsB.recipeName});
+ testCase.assertEqual(recipeNamesB, expectedNames);
+ testCase.assertNumElements(renderingsA, 14);
+ end
+
+ function testFindRenderingsSubset(testCase)
+ renderings = rtbFindRenderings(testCase.folderA, ...
+ 'filter', 'CoordinatesTest[^\.]+\.mat$');
+ testCase.assertNumElements(renderings, 2);
+ recipeNames = {renderings.recipeName};
+ testCase.assertEqual(strcmp(recipeNames, 'rtbMakeCoordinatesTest'), true(1,2));
+ end
+
+ function testFindRenderingsNone(testCase)
+ renderings = rtbFindRenderings(testCase.folderA, ...
+ 'filter', 'notagoodpattern');
+ testCase.assertEmpty(renderings);
+ end
+
+ function fetchReferenceSuccess(testCase)
+ renderings = rtbFetchReferenceData('rtbMakeInterreflection', ...
+ 'referenceRoot', testCase.scratchFolder);
+ testCase.assertNumElements(renderings, 6);
+ recipeNames = {renderings.recipeName};
+ testCase.assertEqual(strcmp(recipeNames, 'rtbMakeInterreflection'), true(1,6));
+ end
+
+ function fetchReferenceNone(testCase)
+ renderings = rtbFetchReferenceData('nosuchrecipe', ...
+ 'referenceRoot', testCase.scratchFolder);
+ testCase.assertNumElements(renderings, 0);
+ end
+
+ function compareRenderingToSelf(testCase)
+ renderings = rtbFindRenderings(testCase.folderA);
+ comparison = rtbCompareRenderings(renderings(1), renderings(1));
+ testCase.assertEqual(comparison.corrcoef, 1, 'AbsTol', 1e-6);
+ testCase.assertEqual(comparison.relNormDiff.max, 0,'AbsTol', 1e-6);
+ end
+
+ function compareRenderingToOther(testCase)
+ renderings = rtbFindRenderings(testCase.folderA);
+ comparison = rtbCompareRenderings(renderings(1), renderings(2));
+ testCase.assertLessThan(comparison.corrcoef, 1);
+ testCase.assertGreaterThan(comparison.relNormDiff.max, 0);
+ end
+
+ function compareRenderingPlot(testCase)
+ renderings = rtbFindRenderings(testCase.folderA);
+ comparison = rtbCompareRenderings(renderings(1), renderings(1));
+ fig = rtbPlotRenderingComparison(comparison);
+ close(fig);
+ end
+
+ function compareFolderToSelf(testCase)
+ comparisons = rtbCompareManyRecipes(testCase.folderA, testCase.folderA, ...
+ 'fetchReferenceData', false);
+ testCase.assertNumElements(comparisons, 14);
+ testCase.assertEqual([comparisons.corrcoef], ones(1, 14), 'AbsTol', 1e-6);
+ relNormDiff = [comparisons.relNormDiff];
+ testCase.assertEqual([relNormDiff.max], zeros(1, 14) ,'AbsTol', 1e-6);
+ end
+
+ function compareFolderToOther(testCase)
+ comparisons = rtbCompareManyRecipes(testCase.folderA, testCase.folderB, ...
+ 'fetchReferenceData', false);
+ testCase.assertNumElements(comparisons, 14);
+ % real correlations can be unity, skip correlation test
+ relNormDiff = [comparisons.relNormDiff];
+ testCase.assertGreaterThan([relNormDiff.max], 0);
+ end
+
+ function compareFolderToReference(testCase)
+ comparisons = rtbCompareManyRecipes(testCase.folderA, testCase.scratchFolder, ...
+ 'fetchReferenceData', true);
+ testCase.assertNumElements(comparisons, 14);
+ % real correlations can be unity, skip correlation test
+ relNormDiff = [comparisons.relNormDiff];
+ testCase.assertGreaterThan([relNormDiff.max], 0);
+ end
+
+ function compareFolderPlot(testCase)
+ comparisons = rtbCompareManyRecipes(testCase.folderA, testCase.folderA, ...
+ 'fetchReferenceData', false);
+ fig = rtbPlotManyRecipeComparisons(comparisons);
+ close(fig);
+ end
+
+ function epicComparisonPlots(testCase)
+ [comparisons, ~, figs] = rtbRunEpicComparison(testCase.folderA, testCase.folderA, ...
+ 'plotSummary', true, ...
+ 'plotImages', true);
+ testCase.assertNumElements(comparisons, 14);
+ testCase.assertNumElements(figs, 14 + 1);
+ close(figs);
+ end
+
+ function epicComparisonNoPlots(testCase)
+ [comparisons, ~, figs] = rtbRunEpicComparison(testCase.folderA, testCase.folderA, ...
+ 'plotSummary', false, ...
+ 'plotImages', false);
+ testCase.assertNumElements(comparisons, 14);
+ testCase.assertEmpty(figs);
+ end
+
+ function epicComparisonClosePlots(testCase)
+ [comparisons, ~, figs] = rtbRunEpicComparison(testCase.folderA, testCase.folderA, ...
+ 'plotSummary', true, ...
+ 'plotImages', true, ...
+ 'closeSummary', true, ...
+ 'closeImages', true);
+ testCase.assertNumElements(comparisons, 14);
+ testCase.assertEmpty(figs);
+ end
+
+ function epicComparisonSavePlots(testCase)
+ [comparisons, ~, figs] = rtbRunEpicComparison(testCase.folderA, testCase.folderA, ...
+ 'plotSummary', true, ...
+ 'plotImages', true, ...
+ 'closeSummary', true, ...
+ 'closeImages', true, ...
+ 'figureFolder', testCase.scratchFolder, ...
+ 'summaryName', 'test-summary');
+ testCase.assertNumElements(comparisons, 14);
+ testCase.assertEmpty(figs);
+
+ % summary should have been saved in scratch folder
+ summaryFig = fullfile(testCase.scratchFolder, 'test-summary.fig');
+ summaryPng = fullfile(testCase.scratchFolder, 'test-summary.png');
+ testCase.assertEqual(exist(summaryFig, 'file'), 2);
+ testCase.assertEqual(exist(summaryPng, 'file'), 2);
+
+ % images should have been saved in the scratchFolder
+ for cc = 1:numel(comparisons)
+ identifier = comparisons(cc).renderingA.identifier;
+ imageFig = fullfile(testCase.scratchFolder, [identifier '.fig']);
+ imagePng = fullfile(testCase.scratchFolder, [identifier '.png']);
+ testCase.assertEqual(exist(imageFig, 'file'), 2);
+ testCase.assertEqual(exist(imagePng, 'file'), 2);
+ end
+ end
+ end
+end
diff --git a/Test/Automated/RtbExampleTests.m b/Test/Automated/RtbExampleTests.m
index 46bec743d2392c15bb348db96bcfa4116a5c30bd..df8f427ac08e555ff347184ad05fdb2dfb0790cd 100644
--- a/Test/Automated/RtbExampleTests.m
+++ b/Test/Automated/RtbExampleTests.m
@@ -3,6 +3,7 @@ classdef RtbExampleTests < matlab.unittest.TestCase
properties
outputRoot = fullfile(rtbWorkingFolder(), 'RtbExampleTests');
+ referenceRoot = fullfile(rtbWorkingFolder(), 'RtbReference');
end
methods (TestMethodSetup)
@@ -10,13 +11,17 @@ classdef RtbExampleTests < matlab.unittest.TestCase
if 7 == exist(testCase.outputRoot, 'dir')
rmdir(testCase.outputRoot, 's');
end
+
+ if 7 == exist(testCase.referenceRoot, 'dir')
+ rmdir(testCase.referenceRoot, 's');
+ end
end
end
methods (Test)
function testNotAnExample(testCase)
- results = rtbTestAllExampleScenes( ...
+ results = rtbRunEpicTest( ...
'outputRoot', testCase.outputRoot, ...
'makeFunctions', {'notAnExample.m'});
testCase.assertFalse(results.isSuccess);
@@ -24,35 +29,79 @@ classdef RtbExampleTests < matlab.unittest.TestCase
end
function testCoordinatesTest(testCase)
- results = rtbTestAllExampleScenes( ...
+ results = rtbRunEpicTest( ...
'outputRoot', testCase.outputRoot, ...
'makeFunctions', {'rtbMakeCoordinatesTest.m'});
testCase.assertTrue(results.isSuccess);
testCase.assertEmpty(results.error);
+
+ % compare to reference rendering
+ comparisons = rtbCompareManyRecipes( ...
+ testCase.outputRoot, ...
+ testCase.referenceRoot, ...
+ 'fetchReferenceData', true);
+ testCase.assertTrue(all([comparisons.isGoodComparison]));
+ testCase.assertTrue(all([comparisons.corrcoef] > 0.75));
+ relNormDiffs = [comparisons.relNormDiff];
+ testCase.assertTrue(all([relNormDiffs.max] < 2.5));
+ testCase.assertTrue(all([relNormDiffs.mean] < 2.5));
end
function testDragon(testCase)
- results = rtbTestAllExampleScenes( ...
+ results = rtbRunEpicTest( ...
'outputRoot', testCase.outputRoot, ...
'makeFunctions', {'rtbMakeDragon.m'});
testCase.assertTrue(results.isSuccess);
testCase.assertEmpty(results.error);
+
+ % compare to reference rendering
+ comparisons = rtbCompareManyRecipes( ...
+ testCase.outputRoot, ...
+ testCase.referenceRoot, ...
+ 'fetchReferenceData', true);
+ testCase.assertTrue(all([comparisons.isGoodComparison]));
+ testCase.assertTrue(all([comparisons.corrcoef] > 0.75));
+ relNormDiffs = [comparisons.relNormDiff];
+ testCase.assertTrue(all([relNormDiffs.max] < 2.5));
+ testCase.assertTrue(all([relNormDiffs.mean] < 2.5));
end
function testMaterialSphereBumps(testCase)
- results = rtbTestAllExampleScenes( ...
+ results = rtbRunEpicTest( ...
'outputRoot', testCase.outputRoot, ...
'makeFunctions', {'rtbMakeMaterialSphereBumps.m'});
testCase.assertTrue(results.isSuccess);
testCase.assertEmpty(results.error);
+
+ % compare to reference rendering
+ comparisons = rtbCompareManyRecipes( ...
+ testCase.outputRoot, ...
+ testCase.referenceRoot, ...
+ 'fetchReferenceData', true);
+ testCase.assertTrue(all([comparisons.isGoodComparison]));
+ testCase.assertTrue(all([comparisons.corrcoef] > 0.75));
+ relNormDiffs = [comparisons.relNormDiff];
+ testCase.assertTrue(all([relNormDiffs.max] < 2.5));
+ testCase.assertTrue(all([relNormDiffs.mean] < 2.5));
end
function testMaterialSphereRemodeled(testCase)
- results = rtbTestAllExampleScenes( ...
+ results = rtbRunEpicTest( ...
'outputRoot', testCase.outputRoot, ...
'makeFunctions', {'rtbMakeMaterialSphereRemodeled.m'});
testCase.assertTrue(results.isSuccess);
testCase.assertEmpty(results.error);
+
+ % compare to reference rendering
+ comparisons = rtbCompareManyRecipes( ...
+ testCase.outputRoot, ...
+ testCase.referenceRoot, ...
+ 'fetchReferenceData', true);
+ testCase.assertTrue(all([comparisons.isGoodComparison]));
+ testCase.assertTrue(all([comparisons.corrcoef] > 0.75));
+ relNormDiffs = [comparisons.relNormDiff];
+ testCase.assertTrue(all([relNormDiffs.max] < 2.5));
+ testCase.assertTrue(all([relNormDiffs.mean] < 2.5));
end
end
end
diff --git a/Test/Interactive/Comparison/rtbCompareManyRecipes.m b/Test/Interactive/Comparison/rtbCompareManyRecipes.m
new file mode 100644
index 0000000000000000000000000000000000000000..c620716a43bda6feec90449f4488064055c56aa9
--- /dev/null
+++ b/Test/Interactive/Comparison/rtbCompareManyRecipes.m
@@ -0,0 +1,121 @@
+function [comparisons, matchInfo] = rtbCompareManyRecipes(folderA, folderB, varargin)
+%% Compare paris of renderings across two folders.
+%
+% comparisons = rtbCompareManyRecipes(folderA, folderB) finds rendering
+% data files in the given folderA and folderB and attempts to match up
+% pairs of renderings that came from the same recipe and renderer.
+% For each pair, computes difference images and statistics.
+%
+% Returns a struct array of image comparisons, as returned from
+% rtbCompareRenderings().
+%
+% rtbCompareManyRecipes( ... 'fetchReferenceData', fetchReferenceData)
+% specify whether to use Remote Data Toolbox to fetch reference data for
+% comparison. The default is true, fetch reference data when there is a
+% recipe in folderA that was not found in folderB, and cache the fetched
+% data in folderB.
+%
+%%% RenderToolbox4 Copyright (c) 2012-2017 The RenderToolbox Team.
+%%% About Us://github.com/RenderToolbox/RenderToolbox4/wiki/About-Us
+%%% RenderToolbox4 is released under the MIT License. See LICENSE file.
+
+parser = inputParser();
+parser.KeepUnmatched = true;
+parser.addRequired('folderA', @ischar);
+parser.addRequired('folderB', @ischar);
+parser.addParameter('fetchReferenceData', true, @islogical);
+parser.parse(folderA, folderB, varargin{:});
+folderA = parser.Results.folderA;
+folderB = parser.Results.folderB;
+fetchReferenceData = parser.Results.fetchReferenceData;
+
+
+%% Identify renderings and recipes to compare.
+renderingsA = rtbFindRenderings(folderA, varargin{:});
+recipeNames = unique({renderingsA.recipeName});
+nRecipes = numel(recipeNames);
+
+renderingsB = rtbFindRenderings(folderB, varargin{:});
+
+%% Compare one recipe at a time, fetch data as necessary.
+comparisonsCell = cell(1, nRecipes);
+matchInfoCell = cell(1, nRecipes);
+for rr = 1:nRecipes
+ recipeName = recipeNames{rr};
+ fprintf('Comparing renderings for recipe <%s>.\n', recipeName);
+
+ isRecipeA = strcmp({renderingsA.recipeName}, recipeName);
+ recipeRenderingsA = renderingsA(isRecipeA);
+
+ % fetch missing recipe for B?
+ if isempty(renderingsB)
+ isRecipeB = false;
+ else
+ isRecipeB = strcmp({renderingsB.recipeName}, recipeName);
+ end
+
+ if any(isRecipeB)
+ recipeRenderingsB = renderingsB(isRecipeB);
+ elseif fetchReferenceData
+ fprintf(' Fetching reference data to <%s>...\n', folderB);
+ recipeRenderingsB = rtbFetchReferenceData(recipeName, ...
+ 'referenceRoot', folderB, ...
+ varargin{:});
+ if isempty(recipeRenderingsB)
+ fprintf(' ...could not fetch, skipping this recipe.\n');
+ continue;
+ else
+ fprintf(' ...OK.\n');
+ end
+ else
+ fprintf(' Skipping recipe not found in <%s>.\n', folderB);
+ continue;
+ end
+
+ % match pairs of renderings for recipes A and B
+ info = matchRenderingPairs(recipeRenderingsA, recipeRenderingsB);
+ fprintf(' Found %d matched pairs of renderings.\n', info.nPairs);
+
+ % run a comparison for each matched pair
+ pairsCell = cell(1, info.nPairs);
+ for pp = 1:info.nPairs
+ fprintf(' %s.\n', info.matchedA(pp).identifier);
+ pairsCell{pp} = rtbCompareRenderings(info.matchedA(pp), info.matchedB(pp), varargin{:});
+ end
+ comparisonsCell{rr} = [pairsCell{:}];
+ matchInfoCell{rr} = info;
+
+ % report on unmatched renderings
+ if ~isempty(info.unmatchedA)
+ nUnmatched = info.unmatchedA;
+ fprintf(' %d renderings in A were not matched in B:\n', nUnmatched);
+ for uu = 1:nUnmatched
+ fprintf(' %s\n', info.info.unmatchedA(uu).identifier);
+ end
+ end
+
+ if ~isempty(info.unmatchedB)
+ nUnmatched = info.unmatchedB;
+ fprintf(' %d renderings in B were not matched in A:\n', nUnmatched);
+ for uu = 1:nUnmatched
+ fprintf(' %s\n', info.info.unmatchedB(uu).identifier);
+ end
+ end
+end
+comparisons = [comparisonsCell{:}];
+matchInfo = [matchInfoCell{:}];
+
+
+%% For pairs of comparable renderings from two sets.
+function info = matchRenderingPairs(renderingsA, renderingsB)
+identifiersA = {renderingsA.identifier};
+identifiersB = {renderingsB.identifier};
+[~, indexA, indexB] = intersect(identifiersA, identifiersB, 'stable');
+[~, unmatchedIndexA] = setdiff(identifiersA, identifiersB);
+[~, unmatchedIndexB] = setdiff(identifiersB, identifiersA);
+
+info.nPairs = numel(indexA);
+info.matchedA = renderingsA(indexA);
+info.matchedB = renderingsB(indexB);
+info.unmatchedA = renderingsA(unmatchedIndexA);
+info.unmatchedB = renderingsB(unmatchedIndexB);
diff --git a/Test/Interactive/Comparison/rtbCompareRenderings.m b/Test/Interactive/Comparison/rtbCompareRenderings.m
new file mode 100644
index 0000000000000000000000000000000000000000..7ca36617f4c0c11634437b236dda6c28b3a8900c
--- /dev/null
+++ b/Test/Interactive/Comparison/rtbCompareRenderings.m
@@ -0,0 +1,194 @@
+function comparison = rtbCompareRenderings(renderingA, renderingB, varargin)
+%% Compare two renderings for difference images and statistics.
+%
+% comparison = rtbCompareRenderings(renderingA, renderingB) compares the
+% given renderingA against the given renderingB. Each must be a rendering
+% record as returned from rtbRenderingRecord() or rtbFindRenderings().
+% Returns a comparison struct with many various fields including:
+% - A -- the multispectral image from renderingA
+% - B -- the multispectral image from renderingB
+% - aMinusB -- the multispectral difference image A - B
+% - bMinusA -- the multispectral difference image B - A
+% - relNormDiff -- the min, mean, and max of per-pixel difference
+% - corrcoef -- the overall correlation between A and B
+% - error -- any error encountered during comparisons
+%
+% The relNormDiff is obtained by normalizing A and B each by its max value,
+% taking the absolute difference, and dividing the difference by the
+% normalized values of A. Finally, values of relNormDiff that are less
+% than the given denominatorThreshold are set to nan. This is to avoid
+% unfair comparisons when dividing by small values.
+%
+% rtbCompareRenderings( ... 'denominatorThreshold', denominatorThreshold)
+% specify the denominatorThreshold to use when computing the relNormDiff
+% image described above. The default is 0.2.
+%
+%%% RenderToolbox4 Copyright (c) 2012-2017 The RenderToolbox Team.
+%%% About Us://github.com/RenderToolbox/RenderToolbox4/wiki/About-Us
+%%% RenderToolbox4 is released under the MIT License. See LICENSE file.
+
+parser = inputParser();
+parser.KeepUnmatched = true;
+parser.addRequired('renderingA', @isstruct);
+parser.addRequired('renderingB', @isstruct);
+parser.addParameter('denominatorThreshold', 0.2, @isnumeric);
+parser.parse(renderingA, renderingB, varargin{:});
+renderingA = parser.Results.renderingA;
+renderingB = parser.Results.renderingB;
+denominatorThreshold = parser.Results.denominatorThreshold;
+
+
+%% Consistent struct fields for output.
+comparison = parser.Results;
+comparison.error = '';
+comparison.samplingA = [];
+comparison.samplingB = [];
+comparison.isGoodComparison = false;
+comparison.subpixelsA = [];
+comparison.subpixelsB = [];
+comparison.normA = [];
+comparison.normB = [];
+comparison.normDiff = [];
+comparison.absNormDiff = [];
+comparison.relNormDiff = [];
+comparison.corrcoef = [];
+comparison.A = [];
+comparison.B = [];
+comparison.aMinusB = [];
+comparison.bMinusA = [];
+
+
+%% Load multispectral images.
+dataA = load(renderingA.fileName);
+dataB = load(renderingB.fileName);
+
+% do we have images?
+hasImageA = isfield(dataA, 'multispectralImage');
+hasImageB = isfield(dataB, 'multispectralImage');
+if hasImageA
+ if hasImageB
+ % each has an image -- proceed
+ else
+ comparison.error = ...
+ 'Data file A has a multispectralImage but B does not.';
+ disp(comparison.error);
+ return;
+ end
+else
+ if hasImageB
+ comparison.error = ...
+ 'Data file B has a multispectralImage but A does not.';
+ disp(comparison.error);
+ return;
+ else
+ comparison.error = ...
+ 'Neither data file A nor B has a multispectralImage.';
+ disp(comparison.error);
+ return;
+ end
+end
+multispectralA = dataA.multispectralImage;
+multispectralB = dataB.multispectralImage;
+
+
+%% Sanity check image dimensions.
+if ~isequal(size(multispectralA, 1), size(multispectralB, 1)) ...
+ || ~isequal(size(multispectralA, 2), size(multispectralB, 2))
+
+ comparison.error = ...
+ sprintf('Image A[%s] is not the same size as image B[%s].', ...
+ num2str(size(multispectralA)), num2str(size(multispectralB)));
+ disp(comparison.error);
+ return;
+end
+
+
+%% Sanity check spectral sampling.
+hasSamplingA = isfield(dataA, 'S');
+hasSamplingB = isfield(dataB, 'S');
+if hasSamplingA
+ if hasSamplingB
+ % each has a sampling "S" -- proceed
+ else
+ comparison.error = ...
+ 'Data file A has a spectral sampling "S" but B does not.';
+ disp(comparison.error);
+ return;
+ end
+else
+ if hasSamplingB
+ comparison.error = ...
+ 'Data file B has a spectral sampling "S" but A does not.';
+ disp(comparison.error);
+ return;
+ else
+ comparison.error = ...
+ 'Neither data file A nor B has spectral sampling "S".';
+ return;
+ end
+end
+comparison.samplingA = dataA.S;
+comparison.samplingB = dataB.S;
+
+% do spectral samplings agree?
+if ~isequal(dataA.S, dataB.S)
+ comparison.error = ...
+ sprintf('Spectral sampling A[%s] is not the same as B[%s].', ...
+ num2str(dataA.S), num2str(dataB.S));
+ disp(comparison.error);
+ % proceed with comparison, despite sampling mismatch
+end
+
+% match images based on sampling depths
+[multispectralA, multispectralB] = truncatePlanes( ...
+ multispectralA, multispectralB, dataA.S, dataB.S);
+
+
+%% Sanity Checks OK.
+comparison.isGoodComparison = true;
+
+
+%% Per-pixel difference stats.
+normA = multispectralA / max(multispectralA(:));
+normB = multispectralB / max(multispectralB(:));
+normDiff = normA - normB;
+absNormDiff = abs(normDiff);
+relNormDiff = absNormDiff ./ normA;
+relNormDiff(normA < denominatorThreshold) = nan;
+
+% summarize differnece stats
+comparison.subpixelsA = summarizeData(multispectralA);
+comparison.subpixelsB = summarizeData(multispectralB);
+comparison.normA = summarizeData(normA);
+comparison.normB = summarizeData(normB);
+comparison.normDiff = summarizeData(normDiff);
+comparison.absNormDiff = summarizeData(absNormDiff);
+comparison.relNormDiff = summarizeData(relNormDiff);
+
+
+%% Overall correlation.
+r = corrcoef(multispectralA(:), multispectralB(:));
+comparison.corrcoef = r(1, 2);
+
+
+%% Difference images.
+comparison.A = multispectralA;
+comparison.B = multispectralB;
+comparison.aMinusB = multispectralA - multispectralB;
+comparison.bMinusA = multispectralB - multispectralA;
+
+
+%% Truncate spectral planes if one image has more planes.
+function [truncA, truncB, truncSampling] = truncatePlanes(A, B, samplingA, samplingB)
+nPlanes = min(samplingA(3), samplingB(3));
+truncSampling = [samplingA(1:2) nPlanes];
+truncA = A(:,:,1:nPlanes);
+truncB = B(:,:,1:nPlanes);
+
+
+%% Summarize a distribuition of data with a struct of stats.
+function summary = summarizeData(data)
+finiteData = data(isfinite(data));
+summary.min = min(finiteData);
+summary.mean = mean(finiteData);
+summary.max = max(finiteData);
diff --git a/Test/Interactive/rtbFetchReferenceData.m b/Test/Interactive/Comparison/rtbFetchReferenceData.m
similarity index 89%
rename from Test/Interactive/rtbFetchReferenceData.m
rename to Test/Interactive/Comparison/rtbFetchReferenceData.m
index 8b400f66c862002c233f8a601277efbe7a9f58aa..adca9695739b4a1568e18a11029764114c8b1172 100644
--- a/Test/Interactive/rtbFetchReferenceData.m
+++ b/Test/Interactive/Comparison/rtbFetchReferenceData.m
@@ -30,6 +30,7 @@ function [renderings, referenceRoot, artifact] = rtbFetchReferenceData(recipeNam
%%% RenderToolbox4 is released under the MIT License. See LICENSE file.
parser = inputParser();
+parser.KeepUnmatched = true;
parser.addRequired('recipeName', @ischar);
parser.addParameter('rdtConfig', 'render-toolbox');
parser.addParameter('remotePath', 'reference-data', @ischar);
@@ -42,12 +43,17 @@ remotePath = parser.Results.remotePath;
referenceVersion = parser.Results.referenceVersion;
referenceRoot = parser.Results.referenceRoot;
-
%% Get a whole recipe from the server.
artifactPath = fullfile(remotePath, recipeName);
-[fileName, artifact] = rdtReadArtifact(rdtConfig, artifactPath, recipeName, ...
- 'version', referenceVersion, ...
- 'type', 'zip');
+try
+ [fileName, artifact] = rdtReadArtifact(rdtConfig, artifactPath, recipeName, ...
+ 'version', referenceVersion, ...
+ 'type', 'zip');
+catch err
+ renderings = [];
+ artifact = [];
+ return;
+end
if isempty(fileName)
renderings = [];
@@ -58,6 +64,9 @@ end
%% Explode renderings it into the destination folder.
destination = fullfile(referenceRoot, recipeName);
+if 7 ~= exist(destination, 'dir')
+ mkdir(destination);
+end
unzip(fileName, destination);
% scan for rendering records
diff --git a/Test/Interactive/rtbFindRenderings.m b/Test/Interactive/Comparison/rtbFindRenderings.m
similarity index 83%
rename from Test/Interactive/rtbFindRenderings.m
rename to Test/Interactive/Comparison/rtbFindRenderings.m
index cdde19cb50fced1a4ba722f4c9b6fd6ca2b2adaa..274c18179ec0dea76d42a3512a2f63af0842d140 100644
--- a/Test/Interactive/rtbFindRenderings.m
+++ b/Test/Interactive/Comparison/rtbFindRenderings.m
@@ -32,6 +32,7 @@ function renderings = rtbFindRenderings(rootFolder, varargin)
%%% RenderToolbox4 is released under the MIT License. See LICENSE file.
parser = inputParser();
+parser.KeepUnmatched = true;
parser.addRequired('rootFolder', @ischar);
parser.addParameter('filter', '\.mat$', @ischar);
parser.addParameter('renderingsFolderName', 'renderings', @ischar);
@@ -63,20 +64,31 @@ for ff = 1:nFiles
renderingsFolderIndex = find(isRenderingsFolder, 1, 'last');
% recipe name comes just before renderingsFolderName
- recipeName = pathParts{renderingsFolderIndex - 1};
+ recipeNameIndex = renderingsFolderIndex - 1;
+ recipeName = pathParts{recipeNameIndex};
% renderer name comes just after renderingsFolderName, if any
- if nPathParts > renderingsFolderIndex
- rendererName = pathParts{renderingsFolderIndex + 1};
+ rendererNameIndex = renderingsFolderIndex + 1;
+ if nPathParts >= rendererNameIndex
+ rendererName = pathParts{rendererNameIndex};
else
rendererName = '';
end
+ % the path leading up to the recipeNamem, if any, is where it came from
+ sourceFolderIndices = 1:(recipeNameIndex-1);
+ if isempty(sourceFolderIndices)
+ sourceFolder = '';
+ else
+ sourceFolder = fullfile(pathParts{sourceFolderIndices});
+ end
+
renderingsCell{ff} = rtbRenderingRecord( ...
'recipeName', recipeName, ...
'rendererName', rendererName, ...
'imageNumber', imageNumber, ...
'imageName', imageName, ...
- 'fileName', files{ff});
+ 'fileName', files{ff}, ...
+ 'sourceFolder', sourceFolder);
end
renderings = [renderingsCell{:}];
diff --git a/Test/Interactive/Comparison/rtbPlotManyRecipeComparisons.m b/Test/Interactive/Comparison/rtbPlotManyRecipeComparisons.m
new file mode 100644
index 0000000000000000000000000000000000000000..72ef80c8d14779d9d8a13c8057b6acadc4073f32
--- /dev/null
+++ b/Test/Interactive/Comparison/rtbPlotManyRecipeComparisons.m
@@ -0,0 +1,134 @@
+function fig = rtbPlotManyRecipeComparisons(comparisons, varargin)
+%% Plot a many recipe comparisons from rtbCompareManyRecipes().
+%
+% fig = fig = rtbPlotManyRecipeComparisons(comparisons) makes a plot to
+% visualize the given struct array of comparison results, as produced by
+% rtbCompareManyRecipes().
+%
+%%% RenderToolbox4 Copyright (c) 2012-2017 The RenderToolbox Team.
+%%% About Us://github.com/RenderToolbox/RenderToolbox4/wiki/About-Us
+%%% RenderToolbox4 is released under the MIT License. See LICENSE file.
+
+parser = inputParser();
+parser.KeepUnmatched = true;
+parser.addRequired('comparisons', @isstruct);
+parser.addParameter('fig', figure());
+parser.addParameter('figureWidth', 1000, @isnumeric);
+parser.addParameter('nRows', 25, @isnumeric);
+parser.addParameter('correlationMin', 0.8, @isnumeric);
+parser.addParameter('correlationStep', 0.05, @isnumeric);
+parser.addParameter('errorMax', 3, @isnumeric);
+parser.addParameter('errorStep', 0.5, @isnumeric);
+parser.parse(comparisons, varargin{:});
+comparisons = parser.Results.comparisons;
+fig = parser.Results.fig;
+figureWidth = parser.Results.figureWidth;
+nRows = parser.Results.nRows;
+correlationMin = parser.Results.correlationMin;
+correlationStep = parser.Results.correlationStep;
+errorMax = parser.Results.errorMax;
+errorStep = parser.Results.errorStep;
+
+
+%% Set up the figure.
+figureName = sprintf('Summary of %d rendering comparisons', ...
+ numel(comparisons));
+set(fig, ...
+ 'Name', figureName, ...
+ 'NumberTitle', 'off');
+
+position = get(fig, 'Position');
+if position(3) < figureWidth
+ position(3) = figureWidth;
+ set(fig, 'Position', position);
+end
+
+%% Summary of correlation coefficients.
+correlationTicks = correlationMin : correlationStep : 1;
+correlationTickLabels = num2cell(correlationTicks);
+correlationTickLabels{1} = '<=';
+correlation = [comparisons.corrcoef];
+correlation(correlation < correlationMin) = correlationMin;
+
+renderingsA = [comparisons.renderingA];
+names = {renderingsA.identifier};
+nLines = numel(names);
+ax(1) = subplot(1, 3, 2, ...
+ 'Parent', fig, ...
+ 'YTick', 1:nLines, ...
+ 'YTickLabel', names, ...
+ 'YGrid', 'on', ...
+ 'XLim', [correlationTicks(1), correlationTicks(end)], ...
+ 'XTick', correlationTicks, ...
+ 'XTickLabel', correlationTickLabels);
+line(correlation, 1:nLines, ...
+ 'Parent', ax(1), ...
+ 'LineStyle', 'none', ...
+ 'Marker', 'o', ...
+ 'Color', [0 0 1])
+xlabel(ax(1), 'correlation');
+
+
+%% Overall title.
+name = sprintf('%s vs %s', ...
+ comparisons(1).renderingA.sourceFolder, ...
+ comparisons(1).renderingB.sourceFolder);
+title(ax(1), name, 'Interpreter', 'none');
+
+
+%% Summary of mean and max subpixel differences.
+errorTicks = 0 : errorStep : errorMax;
+errorTickLabels = num2cell(errorTicks);
+errorTickLabels{end} = '>=';
+
+relNormDiff = [comparisons.relNormDiff];
+maxes = [relNormDiff.max];
+means = [relNormDiff.mean];
+maxes(maxes > errorMax) = errorMax;
+means(means > errorMax) = errorMax;
+ax(2) = subplot(1, 3, 3, ...
+ 'Parent', fig, ...
+ 'YTick', 1:nLines, ...
+ 'YTickLabel', 1:nLines, ...
+ 'YAxisLocation', 'right', ...
+ 'YGrid', 'on', ...
+ 'XLim', [errorTicks(1), errorTicks(end)], ...
+ 'XTick', errorTicks, ...
+ 'XTickLabel', errorTickLabels);
+line(maxes, 1:nLines, ...
+ 'Parent', ax(2), ...
+ 'LineStyle', 'none', ...
+ 'Marker', '+', ...
+ 'Color', [1 0 0])
+line(means, 1:nLines, ...
+ 'Parent', ax(2), ...
+ 'LineStyle', 'none', ...
+ 'Marker', 'o', ...
+ 'Color', [0 0 0])
+legend(ax(2), 'max', 'mean', 'Location', 'northeast');
+xlabel(ax(2), 'relative diff');
+
+
+%% Let the user scroll both axes at the same time.
+scrollerData.axes = ax;
+scrollerData.nLinesAtATime = nRows;
+scroller = uicontrol( ...
+ 'Parent', fig, ...
+ 'Units', 'normalized', ...
+ 'Position', [.95 0 .05 1], ...
+ 'Callback', @scrollAxes, ...
+ 'Min', 1, ...
+ 'Max', max(2, nLines), ...
+ 'Value', nLines, ...
+ 'Style', 'slider', ...
+ 'SliderStep', [1 2], ...
+ 'UserData', scrollerData);
+scrollAxes(scroller, []);
+
+
+%% Scroll summary axes together.
+function scrollAxes(object, event)
+scrollerData = get(object, 'UserData');
+topLine = get(object, 'Value');
+yLimit = topLine + [-scrollerData.nLinesAtATime 1];
+set(scrollerData.axes, 'YLim', yLimit);
diff --git a/Test/Interactive/Comparison/rtbPlotRenderingComparison.m b/Test/Interactive/Comparison/rtbPlotRenderingComparison.m
new file mode 100644
index 0000000000000000000000000000000000000000..185bb1e0d2d19a6223f5ecdb180f4d8c78b38a75
--- /dev/null
+++ b/Test/Interactive/Comparison/rtbPlotRenderingComparison.m
@@ -0,0 +1,64 @@
+function fig = rtbPlotRenderingComparison(comparison, varargin)
+%% Plot a rendering comparison from rtbCompareRenderings().
+%
+% fig = rtbPlotRenderingComparison(comparison) makes a plot to visualize
+% the given struct of comparison results, as produced by
+% rtbCompareRenderings().
+%
+%%% RenderToolbox4 Copyright (c) 2012-2017 The RenderToolbox Team.
+%%% About Us://github.com/RenderToolbox/RenderToolbox4/wiki/About-Us
+%%% RenderToolbox4 is released under the MIT License. See LICENSE file.
+
+parser = inputParser();
+parser.KeepUnmatched = true;
+parser.addRequired('comparison', @isstruct);
+parser.addParameter('isScale', true, @islogical);
+parser.addParameter('toneMapFactor', 0, @isnumeric);
+parser.addParameter('fig', figure());
+parser.parse(comparison, varargin{:});
+comparison = parser.Results.comparison;
+isScale = parser.Results.isScale;
+toneMapFactor = parser.Results.toneMapFactor;
+fig = parser.Results.fig;
+
+
+%% Compute RGB images for viewing.
+S = comparison.samplingA;
+rgbA = rtbMultispectralToSRGB(comparison.A, S, ...
+ 'toneMapFactor', toneMapFactor, 'isScale', isScale);
+rgbB = rtbMultispectralToSRGB(comparison.B, S, ...
+ 'toneMapFactor', toneMapFactor, 'isScale', isScale);
+rgbAminusB = rtbMultispectralToSRGB(comparison.aMinusB, S, ...
+ 'toneMapFactor', toneMapFactor, 'isScale', isScale);
+rgbBminusA = rtbMultispectralToSRGB(comparison.bMinusA, S, ...
+ 'toneMapFactor', toneMapFactor, 'isScale', isScale);
+
+
+%% Make the plot.
+name = sprintf('%s isScale %d toneMapFactor %.2f', ...
+ comparison.renderingA.identifier, ...
+ isScale, ...
+ toneMapFactor);
+set(fig, ...
+ 'Name', name, ...
+ 'NumberTitle', 'off');
+
+ax = subplot(2, 2, 2, 'Parent', fig);
+imshow(uint8(rgbA), 'Parent', ax);
+title(ax, 'A');
+xlabel(ax, comparison.renderingA.sourceFolder, ...
+ 'Interpreter', 'none');
+
+ax = subplot(2, 2, 3, 'Parent', fig);
+imshow(uint8(rgbB), 'Parent', ax);
+title(ax, 'B');
+xlabel(ax, comparison.renderingB.sourceFolder, ...
+ 'Interpreter', 'none');
+
+ax = subplot(2, 2, 1, 'Parent', fig);
+imshow(uint8(rgbAminusB), 'Parent', ax);
+title(ax, 'A - B');
+
+ax = subplot(2, 2, 4, 'Parent', fig);
+imshow(uint8(rgbBminusA), 'Parent', ax);
+title(ax, 'B - A');
diff --git a/Test/Interactive/rtbRenderingRecord.m b/Test/Interactive/Comparison/rtbRenderingRecord.m
similarity index 97%
rename from Test/Interactive/rtbRenderingRecord.m
rename to Test/Interactive/Comparison/rtbRenderingRecord.m
index ce806872ccb708c241933fe53fd713f0bd8e678d..db000e3dc45b2db96ad0e1eb4d93e7a6ee8e8863 100644
--- a/Test/Interactive/rtbRenderingRecord.m
+++ b/Test/Interactive/Comparison/rtbRenderingRecord.m
@@ -24,6 +24,7 @@ parser.addParameter('rendererName', '', @ischar);
parser.addParameter('imageNumber', [], @isnumeric);
parser.addParameter('imageName', '',@ischar);
parser.addParameter('fileName', '', @ischar);
+parser.addParameter('sourceFolder', '', @ischar);
parser.parse(varargin{:});
% let the parser do most of the work
diff --git a/Test/Interactive/Comparison/rtbRunEpicComparison.m b/Test/Interactive/Comparison/rtbRunEpicComparison.m
new file mode 100644
index 0000000000000000000000000000000000000000..5ea42c274ae7a849ec309c9de81cfe21a8fb674c
--- /dev/null
+++ b/Test/Interactive/Comparison/rtbRunEpicComparison.m
@@ -0,0 +1,145 @@
+function [comparisons, matchInfo, figs] = rtbRunEpicComparison(folderA, folderB, varargin)
+%% Compare sets of renderings for similarity.
+%
+% comparisons = rtbRunEpicComparison(folderA, folderB) locates renderings
+% in folderA and folderB, compares pairs of renderings found between the
+% two folders, and plots a summary of the comparisons. Returns a struct
+% array of comparison results, one for each pair of renderings.
+%
+% Also returns a struct array of info about how renderings were matched
+% between folderA and folderB, including renderings from each folder that
+% were unmatched.
+%
+% Also returns an array of figure handles for visualizations of the
+% comparison results.
+%
+% rtbRunEpicComparison( ... 'plotSummary', plotSummary) whether to create a
+% plot summarizing the overall comparison results. The default is true,
+% make a summary plot.
+%
+% rtbRunEpicComparison( ... 'closeSummary', closeSummary) whether to create
+% a close the overalls summary plot when done. This is useful when you
+% specify a figureFolder, where the summary plot can be saved to disk. the
+% default is false, don't close the summary figure.
+%
+% rtbRunEpicComparison( ... 'plotImages', plotImages) whether to create a
+% plot showing images and difference images for each pair of renderings.
+% The default is false, don't show the images for each pair.
+%
+% rtbRunEpicComparison( ... 'closeImages', closeImages) whether to close
+% the image plot for each pair of renderings when done. This is useful
+% when you specify a figureFolder, where the images can be saved to disk.
+% The default is true, do close the image figures.
+%
+% rtbRunEpicComparison( ... 'figureFolder', figureFolder) specify a folder
+% where generated plots should be saved to disk, as a .fig file and as a
+% .png file. The default is '', don't save figures to disk.
+%
+%%% RenderToolbox4 Copyright (c) 2012-2017 The RenderToolbox Team.
+%%% About Us://github.com/RenderToolbox/RenderToolbox4/wiki/About-Us
+%%% RenderToolbox4 is released under the MIT License. See LICENSE file.
+
+parser = inputParser();
+parser.addRequired('folderA', @ischar);
+parser.addRequired('folderB', @ischar);
+parser.addParameter('plotSummary', true, @islogical);
+parser.addParameter('closeSummary', false, @islogical);
+parser.addParameter('plotImages', true, @islogical);
+parser.addParameter('closeImages', false, @islogical);
+parser.addParameter('figureFolder', '', @ischar);
+parser.addParameter('summaryName', 'epic-summary', @ischar);
+parser.parse(folderA, folderB, varargin{:});
+folderA = parser.Results.folderA;
+folderB = parser.Results.folderB;
+plotSummary = parser.Results.plotSummary;
+closeSummary = parser.Results.closeSummary;
+plotImages = parser.Results.plotImages;
+closeImages = parser.Results.closeImages;
+figureFolder = parser.Results.figureFolder;
+summaryName = parser.Results.summaryName;
+
+figs = [];
+
+
+%% Run the grand comparison.
+[comparisons, matchInfo] = rtbCompareManyRecipes(folderA, folderB, ...
+ varargin{:});
+
+
+%% Sort the summary by size of error.
+goodComparisons = comparisons([comparisons.isGoodComparison]);
+relNormDiff = [goodComparisons.relNormDiff];
+errorStat = [relNormDiff.max];
+[~, order] = sort(errorStat);
+goodComparisons = goodComparisons(order);
+
+
+%% Plot the summary.
+if plotSummary
+ summaryFig = rtbPlotManyRecipeComparisons(goodComparisons, ...
+ varargin{:});
+
+ if ~isempty(figureFolder);
+ imageFileName = fullfile(figureFolder, summaryName);
+ saveFigure(summaryFig, imageFileName);
+ end
+
+ if closeSummary
+ close(summaryFig);
+ else
+ figs = [figs summaryFig];
+ end
+end
+
+
+%% Plot the detail images for each rendering.
+if plotImages
+ nComparisons = numel(goodComparisons);
+ imageFigs = cell(1, nComparisons);
+ for cc = 1:nComparisons
+ imageFig = rtbPlotRenderingComparison(goodComparisons(cc), ...
+ varargin{:});
+
+ if ~isempty(figureFolder);
+ identifier = goodComparisons(cc).renderingA.identifier;
+ imageFileName = fullfile(figureFolder, identifier);
+ saveFigure(imageFig, imageFileName);
+ end
+
+ if closeImages
+ close(imageFig);
+ else
+ imageFigs{cc} = imageFig;
+ end
+ end
+ figs = [figs imageFigs{:}];
+end
+
+
+%% Save a figure to file, watch out for things like uicontrols.
+function saveFigure(fig, fileName)
+
+% flush pending drawing commands
+drawnow();
+
+% hide uicontrols, which can't always be saved
+controls = findobj(fig, 'Type', 'uicontrol');
+set(controls, 'Visible', 'off');
+
+% make sure output location exists
+[filePath, fileName] = fileparts(fileName);
+if 7 ~= exist(filePath, 'dir')
+ mkdir(filePath);
+end
+
+% save a png and a figure
+figName = fullfile(filePath, [fileName '.fig']);
+saveas(fig, figName, 'fig');
+
+pngName = fullfile(filePath, [fileName '.png']);
+set(fig, 'PaperPositionMode', 'auto');
+saveas(fig, pngName, 'png');
+
+% restore uicontrols
+set(controls, 'Visible', 'on');
+
diff --git a/Test/Interactive/rtbCompareAllExampleScenes.m b/Test/Interactive/rtbCompareAllExampleScenes.m
deleted file mode 100644
index e0ec06ba9f59256a80f9b0c5fef489b34c1ec45c..0000000000000000000000000000000000000000
--- a/Test/Interactive/rtbCompareAllExampleScenes.m
+++ /dev/null
@@ -1,565 +0,0 @@
-function [matchInfo, unmatchedA, unmatchedB] = rtbCompareAllExampleScenes(workingFolderA, workingFolderB, varargin)
-%% Compare recipe renderings that were generated at different times.
-%
-% [matchInfo, unmatchedA, unmatchedB] = rtbCompareAllExampleScenes(workingFolderA, workingFolderB)
-% Finds 2 sets of rendering outputs: set A includes renderings located
-% under the given workingFolderA, set B includes renderings located
-% under workingFolderB. Attempts to match up data files from both sets,
-% based on recipe names and renderer names. Computes comparison statistics
-% and shows difference images for each matched pair.
-%
-% Data sets must use a particular folder structure, consistent with
-% rtbWorkingFolder(). For each rendering data file, the expected path is:
-% workingFolder/recipeName/rendererName/renderings/fileName.mat
-%
-% where:
-% - workingFolder is either workingFolderA or workingFolderB
-% - recipeName must be the name of a recipe such as "MakeDragon"
-% - rendererName must be the name of a renderer, like "PBRT" or "Mitsuba"
-% - fileName must be the name of a multi-spectral data file, such as "Dragon-001"
-%
-% rtbCompareAllExampleScenes( ... 'filterExpression', filterExpression)
-% uses the given regular expression filterExpr'/home/ben/Desktop/testA'ession to select file paths.
-% Only data files whose paths match the expression will be compared. The
-% default is to do no such filtering.
-%
-% rtbCompareAllExampleScenes( ... 'visualize', visualize) specifies the
-% level of visualization to do during comparinsons. The options are:
-% - 0 -- don't plot anything
-% - 1 -- (default) plot a summary figure at the end
-% - 2 -- plot a summary figure at the end and a detail figure for each comparison
-%
-% The summary figure will contain two plots:
-%
-% A "correlation" plot will show this correlation betweeen paired
-% multi-spectral images, with each image simply treated as a matrix of
-% numbers.
-%
-% A "relative diff" plot will show the relative differences between paired
-% pixel components, where raw diff is
-% diff = |a-b|/a, where a/max(a) > .2
-% The diff is only calculated when the "a" value is not small, to avoid
-% unfair comparison due to large denominator. For each pair of images, the
-% plot will show the mean and max of the diffs from all pixel components.
-%
-% rtbCompareAllExampleScenes( ... 'figureFolder', figureFolder) specifies
-% an output folder where to save figures used for visualization. The
-% default is rtbWorkingFolder().
-%
-% This function is intended to help validate RenderToolbox installations
-% and detect bugs in the RenderToolbox code. A potential use would
-% compare renderings produced locally with archived renderings located at
-% Amazon S3. For example:
-% % produce renderings locally
-% rtbTestAllExampleScenes('my/local/renderings');
-%
-% % download archived renderings to 'my/local/archive'
-%
-% % summarize local vs archived renderings
-% workingFolderA = 'my/local/renderings/data';
-% workingFolderA = 'my/local/archive/data';
-% visualize = 1;
-% matchInfo = rtbCompareAllExampleScenes(workingFolderA, workingFolderB, 'visualize', visualize);
-%
-% Returns a struct array of info about each matched pair, including file
-% names and differneces between multispectral images (A minus B).
-%
-% Also returns a cell array of paths for files in set A that did not match
-% any of the files in set B. Likewise, returns a cell array of paths for
-% files in set B that did not match any of the files in set A.
-%
-%%% RenderToolbox4 Copyright (c) 2012-2016 The RenderToolbox Team.
-%%% About Us://github.com/RenderToolbox/RenderToolbox4/wiki/About-Us
-%%% RenderToolbox4 is released under the MIT License. See LICENSE file.
-
-parser = inputParser();
-parser.addRequired('workingFolderA', @ischar);
-parser.addRequired('workingFolderB', @ischar);
-parser.addParameter('filterExpression', '', @ischar);
-parser.addParameter('visualize', 1, @isnumeric);
-parser.addParameter('figureFolder', fullfile(rtbWorkingFolder(), 'comparisons'), @ischar);
-parser.parse(workingFolderA, workingFolderB, varargin{:});
-workingFolderA = parser.Results.workingFolderA;
-workingFolderB = parser.Results.workingFolderB;
-filterExpression = parser.Results.filterExpression;
-visualize = parser.Results.visualize;
-figureFolder = parser.Results.figureFolder;
-
-matchInfo = [];
-unmatchedA = {};
-unmatchedB = {};
-
-% find .mat files for sets A and B
-fileFilter = [filterExpression '[^\.]*\.mat$'];
-filesA = rtbFindFiles('root', workingFolderA, 'filter', fileFilter);
-filesB = rtbFindFiles('root', workingFolderB, 'filter', fileFilter);
-
-if isempty(filesA)
- fprintf('Found no files for set A in: %s\n', workingFolderA);
- return;
-end
-
-if isempty(filesB)
- fprintf('Found no files for set B in: %s\n', workingFolderB);
- return;
-end
-
-% parse out expected path parts for each file
-infoA = scanDataPaths(filesA);
-infoB = scanDataPaths(filesB);
-
-% report unmatched files
-matchTokensA = {infoA.matchToken};
-matchTokensB = {infoB.matchToken};
-[~, indexA, indexB] = intersect( ...
- matchTokensA, matchTokensB, 'stable');
-[~, unmatchedIndex] = setdiff(matchTokensA, matchTokensB);
-unmatchedA = filesA(unmatchedIndex);
-[~, unmatchedIndex] = setdiff(matchTokensB, matchTokensA);
-unmatchedB = filesB(unmatchedIndex);
-
-if isempty(indexA) || isempty(indexB)
- fprintf('Could not find any file matches.\n');
- return;
-end
-
-% allocate an info struct for image comparisons
-filesA = {infoA.original};
-filesB = {infoB.original};
-matchInfo = struct( ...
- 'fileA', filesA(indexA), ...
- 'fileB', filesB(indexB), ...
- 'workingFolderA', workingFolderA, ...
- 'workingFolderB', workingFolderB, ...
- 'relativePathA', {infoA(indexA).relativePath}, ...
- 'relativePathB', {infoB(indexB).relativePath}, ...
- 'matchTokenA', matchTokensA(indexA), ...
- 'matchTokenB', matchTokensB(indexB), ...
- 'samplingA', [], ...
- 'samplingB', [], ...
- 'denominatorThreshold', 0.2, ...
- 'subpixelsA', [], ...
- 'subpixelsB', [], ...
- 'normA', [], ...
- 'normB', [], ...
- 'normDiff', [], ...
- 'absNormDiff', [], ...
- 'relNormDiff', [], ...
- 'corrcoef', nan, ...
- 'isGoodComparison', false, ...
- 'detailFigure', nan, ...
- 'error', '');
-
-% any comparisons to make?
-nMatches = numel(matchInfo);
-if nMatches > 0
- fprintf('Found %d matched pairs of data files.\n', nMatches);
- fprintf('Some of these might not contain images and would be skipped.\n');
-else
- fprintf('Found no matched pairs.\n');
- return;
-end
-fprintf('\n')
-
-nUnmatchedA = numel(unmatchedA);
-if nUnmatchedA > 0
- fprintf('%d data files in set A had no match in set B:\n', nUnmatchedA);
- for ii = 1:nUnmatchedA
- fprintf(' %s\n', unmatchedA{ii});
- end
-end
-fprintf('\n')
-
-nUnmatchedB = numel(unmatchedB);
-if nUnmatchedB > 0
- fprintf('%d data files in set B had no match in set A:\n', nUnmatchedB);
- for ii = 1:nUnmatchedB
- fprintf(' %s\n', unmatchedB{ii});
- end
-end
-fprintf('\n')
-
-% compare matched images!
-for ii = 1:nMatches
- fprintf('%d of %d: %s\n', ii, nMatches, matchInfo(ii).matchTokenA);
-
- % load data
- dataA = load(matchInfo(ii).fileA);
- dataB = load(matchInfo(ii).fileB);
-
- % do we have images?
- hasImageA = isfield(dataA, 'multispectralImage');
- hasImageB = isfield(dataB, 'multispectralImage');
- if hasImageA
- if hasImageB
- % each has an image -- proceed
- else
- matchInfo(ii).error = ...
- 'Data file A has a multispectralImage but B does not!';
- disp(matchInfo(ii).error);
- continue;
- end
- else
- if hasImageB
- matchInfo(ii).error = ...
- 'Data file B has a multispectralImage but A does not!';
- disp(matchInfo(ii).error);
- continue;
- else
- matchInfo(ii).error = ...
- 'Neither data file A nor B has a multispectralImage -- skipping.';
- continue;
- end
- end
- multispectralA = dataA.multispectralImage;
- multispectralB = dataB.multispectralImage;
-
- % do image dimensions agree?
- if ~isequal(size(multispectralA, 1), size(multispectralB, 1)) ...
- || ~isequal(size(multispectralA, 2), size(multispectralB, 2))
- matchInfo(ii).error = ...
- sprintf('Image A[%s] is not the same size as image B[%s].', ...
- num2str(size(multispectralA)), num2str(size(multispectralB)));
- disp(matchInfo(ii).error);
- continue;
- end
-
- % do we have spectral sampling?
- hasSamplingA = isfield(dataA, 'S');
- hasSamplingB = isfield(dataB, 'S');
- if hasSamplingA
- if hasSamplingB
- % each has a sampling "S" -- proceed
- else
- matchInfo(ii).error = ...
- 'Data file A has a spectral sampling "S" but B does not!';
- disp(matchInfo(ii).error);
- continue;
- end
- else
- if hasSamplingB
- matchInfo(ii).error = ...
- 'Data file B has a spectral sampling "S" but A does not!';
- disp(matchInfo(ii).error);
- continue;
- else
- matchInfo(ii).error = ...
- 'Neither data file A nor B has spectral sampling "S" -- skipping.';
- continue;
- end
- end
- matchInfo(ii).samplingA = dataA.S;
- matchInfo(ii).samplingB = dataB.S;
-
- % do spectral samplings agree?
- if ~isequal(dataA.S, dataB.S)
- matchInfo(ii).error = ...
- sprintf('Spectral sampling A[%s] is not the same as B[%s].', ...
- num2str(dataA.S), num2str(dataB.S));
- disp(matchInfo(ii).error);
- % proceed with comparison, despite sampling mismatch
- end
-
- % tolerate different sectral sampling depths
- [A, B] = truncatePlanes(multispectralA, multispectralB, dataA.S, dataB.S);
-
- % comparison passes all sanity checks
- matchInfo(ii).isGoodComparison = true;
-
- % compute per-pixel component difference stats
- normA = A / max(A(:));
- normB = B / max(B(:));
- normDiff = normA - normB;
- absNormDiff = abs(normDiff);
- relNormDiff = absNormDiff ./ normA;
- cutoff = matchInfo(ii).denominatorThreshold;
- relNormDiff(normA < cutoff) = nan;
-
- % summarize differnece stats
- matchInfo(ii).subpixelsA = summarizeData(A);
- matchInfo(ii).subpixelsB = summarizeData(B);
- matchInfo(ii).normA = summarizeData(normA);
- matchInfo(ii).normB = summarizeData(normB);
- matchInfo(ii).normDiff = summarizeData(normDiff);
- matchInfo(ii).absNormDiff = summarizeData(absNormDiff);
- matchInfo(ii).relNormDiff = summarizeData(relNormDiff);
-
- % compute correlation among pixel components
- r = corrcoef(A(:), B(:));
- matchInfo(ii).corrcoef = r(1, 2);
-
- % plot difference image?
- if visualize > 1
- f = showDifferenceImage(matchInfo(ii), A, B);
- matchInfo(ii).detailFigure = f;
-
- % save detail figure to disk
- drawnow();
- [imagePath, imageName] = fileparts(matchInfo(ii).relativePathA);
- imageCompPath = fullfile(figureFolder, imagePath);
- if ~exist(imageCompPath, 'dir')
- mkdir(imageCompPath);
- end
- figName = fullfile(imageCompPath, [imageName '.fig']);
- saveas(f, figName, 'fig');
- pngName = fullfile(imageCompPath, [imageName '.png']);
- set(f, 'PaperPositionMode', 'auto');
- saveas(f, pngName, 'png');
-
- close(f);
- end
-end
-
-nComparisons = sum([matchInfo.isGoodComparison]);
-nSkipped = nMatches - nComparisons;
-fprintf('Compared %d pairs of data files.\n', nComparisons);
-fprintf('Skipped %d pairs of data files, which is not necessarily a problem.\n', nSkipped);
-
-% plot a grand summary?
-if visualize > 0
- f = showDifferenceSummary(matchInfo);
-
- % save summary figure to disk
- if ~exist(figureFolder, 'dir')
- mkdir(figureFolder);
- end
- imageName = sprintf('%s-summary', mfilename());
- figName = fullfile(figureFolder, [imageName '.fig']);
- saveas(f, figName, 'fig');
-
- % some platforms can't pring uicontrols
- controls = findobj(f, 'Type', 'uicontrol');
- set(controls, 'Visible', 'off');
- pngName = fullfile(figureFolder, [imageName '.png']);
- saveas(f, pngName, 'png');
- set(controls, 'Visible', 'on');
-end
-
-if visualize > 1
- fprintf('\nSee comparison images saved in:\n %s\n', figureFolder);
-end
-
-
-% Scan paths for expected parts:
-% root/recipeName/subfolderName/rendererName/fileName.extension
-function info = scanDataPaths(paths)
-n = numel(paths);
-rootPath = cell(1, n);
-relativePath = cell(1, n);
-recipeName = cell(1, n);
-subfolderName = cell(1, n);
-rendererName = cell(1, n);
-fileName = cell(1, n);
-fileNumber = cell(1, n);
-hasNumber = false(1, n);
-matchToken = cell(1, n);
-for ii = 1:n
- % break off the file name
- [parentPath, baseName, extension] = fileparts(paths{ii});
- fileName{ii} = [baseName extension];
- if numel(baseName) >= 4
- fileNumber{ii} = sscanf(baseName(end-3:end), '-%d');
- end
- hasNumber(ii) = ~isempty(fileNumber{ii});
-
- % break out subfolder names
- scanResult = textscan(parentPath, '%s', 'Delimiter', filesep());
- tokens = scanResult{1};
-
- % is there a renderer folder?
- if any(strcmp(tokens{end}, {'PBRT', 'Mitsuba'}))
- rendererName{ii} = tokens{end};
- subfolderNameIndex = numel(tokens) - 1;
- else
- rendererName{ii} = '';
- subfolderNameIndex = numel(tokens);
- end
-
- % get the named subfolder name
- subfolderName{ii} = tokens{subfolderNameIndex};
-
- % get the recipe name
- recipeName{ii} = tokens{subfolderNameIndex-1};
-
- % get the root path
- rootPath{ii} = fullfile(tokens{1:subfolderNameIndex-2});
-
- % build the rootless relative path
- relativePath{ii} = fullfile(recipeName{ii}, subfolderName{ii}, ...
- rendererName{ii}, fileName{ii});
-
- % build a token for matching across file sets
- if strncmp(recipeName{ii}, 'rtb', 3)
- nameBase = recipeName{ii}(4:end);
- else
- nameBase = recipeName{ii};
- end
- matchTokenBase = [nameBase '-' subfolderName{ii} '-' rendererName{ii} '-'];
- if hasNumber(ii)
- matchToken{ii} = [matchTokenBase sprintf('%03d', fileNumber{ii})];
- else
- matchToken{ii} = [matchTokenBase baseName];
- end
-end
-
-info = struct( ...
- 'original', paths, ...
- 'fileName', fileName, ...
- 'fileNumber', fileNumber, ...
- 'hasNumber', hasNumber, ...
- 'rendererName', rendererName, ...
- 'recipeName', recipeName, ...
- 'subfolderName', subfolderName, ...
- 'rootPath', rootPath, ...
- 'relativePath', relativePath, ...
- 'matchToken', matchToken);
-
-
-% Show sRGB images and sRGB difference images
-function f = showDifferenceImage(info, A, B)
-
-% make SRGB images
-[A, B, S] = truncatePlanes(A, B, info.samplingA, info.samplingB);
-isScale = true;
-toneMapFactor = 0;
-imageA = rtbMultispectralToSRGB(A, S, 'toneMapFactor', toneMapFactor, 'isScale', isScale);
-imageB = rtbMultispectralToSRGB(B, S, 'toneMapFactor', toneMapFactor, 'isScale', isScale);
-imageAB = rtbMultispectralToSRGB(A-B, S, 'toneMapFactor', toneMapFactor, 'isScale', isScale);
-imageBA = rtbMultispectralToSRGB(B-A, S, 'toneMapFactor', toneMapFactor, 'isScale', isScale);
-
-% show images in a new figure
-name = sprintf('sRGB scaled: %s', info.matchTokenA);
-f = figure('Name', name, 'NumberTitle', 'off');
-
-ax = subplot(2, 2, 2, 'Parent', f);
-imshow(uint8(imageA), 'Parent', ax);
-title(ax, ['A: ' info.workingFolderA]);
-
-ax = subplot(2, 2, 3, 'Parent', f);
-imshow(uint8(imageB), 'Parent', ax);
-title(ax, ['B: ' info.workingFolderB]);
-
-ax = subplot(2, 2, 1, 'Parent', f);
-imshow(uint8(imageAB), 'Parent', ax);
-title(ax, 'Difference: A - B');
-
-ax = subplot(2, 2, 4, 'Parent', f);
-imshow(uint8(imageBA), 'Parent', ax);
-title(ax, 'Difference: B - A');
-
-
-% Truncate spectral planes if one image has more planes.
-function [truncA, truncB, truncSampling] = truncatePlanes(A, B, samplingA, samplingB)
-nPlanes = min(samplingA(3), samplingB(3));
-truncSampling = [samplingA(1:2) nPlanes];
-truncA = A(:,:,1:nPlanes);
-truncB = B(:,:,1:nPlanes);
-
-
-% Show a summary of all difference images.
-function f = showDifferenceSummary(info)
-figureName = sprintf('A: %s vs B: %s', ...
- info(1).workingFolderA, info(1).workingFolderB);
-f = figure('Name', figureName, 'NumberTitle', 'off');
-
-% summarize only fair comparisions
-goodInfo = info([info.isGoodComparison]);
-
-% sort the summary by size of error
-diffSummary = [goodInfo.relNormDiff];
-errorStat = [diffSummary.max];
-[~, order] = sort(errorStat);
-goodInfo = goodInfo(order);
-
-% summarize data correlation coefficients
-minCorr = 0.85;
-peggedCorr = 0.8;
-corrTicks = [peggedCorr minCorr:0.05:1];
-corrTickLabels = num2cell(corrTicks);
-corrTickLabels{1} = sprintf('<%.2f', minCorr);
-corr = [goodInfo.corrcoef];
-corr(corr < minCorr) = peggedCorr;
-
-names = {goodInfo.matchTokenA};
-nLines = numel(names);
-ax(1) = subplot(1, 3, 2, ...
- 'Parent', f, ...
- 'YTick', 1:nLines, ...
- 'YTickLabel', names, ...
- 'YGrid', 'on', ...
- 'XLim', [corrTicks(1), corrTicks(end)], ...
- 'XTick', corrTicks, ...
- 'XTickLabel', corrTickLabels);
-line(corr, 1:nLines, ...
- 'Parent', ax(1), ...
- 'LineStyle', 'none', ...
- 'Marker', 'o', ...
- 'Color', [0 0 1])
-title(ax(1), 'correlation');
-
-% summarize mean and max subpixel differences
-maxDiff = 2.5;
-peggedDiff = 3;
-diffTicks = [0:0.5:maxDiff peggedDiff];
-diffTickLabels = num2cell(diffTicks);
-diffTickLabels{end} = sprintf('>%.2f', maxDiff);
-
-diffSummary = [goodInfo.relNormDiff];
-maxes = [diffSummary.max];
-means = [diffSummary.mean];
-maxes(maxes > maxDiff) = peggedDiff;
-means(means > maxDiff) = peggedDiff;
-ax(2) = subplot(1, 3, 3, ...
- 'Parent', f, ...
- 'YTick', 1:nLines, ...
- 'YTickLabel', 1:nLines, ...
- 'YAxisLocation', 'right', ...
- 'YGrid', 'on', ...
- 'XLim', [diffTicks(1), diffTicks(end)], ...
- 'XTick', diffTicks, ...
- 'XTickLabel', diffTickLabels);
-line(maxes, 1:nLines, ...
- 'Parent', ax(2), ...
- 'LineStyle', 'none', ...
- 'Marker', '+', ...
- 'Color', [1 0 0])
-line(means, 1:nLines, ...
- 'Parent', ax(2), ...
- 'LineStyle', 'none', ...
- 'Marker', 'o', ...
- 'Color', [0 0 0])
-legend(ax(2), 'max', 'mean', 'Location', 'northeast');
-title(ax(2), 'relative diff');
-
-% let the user scroll both axes at the same time
-nLinesAtATime = 25;
-scrollerData.axes = ax;
-scrollerData.nLinesAtATime = nLinesAtATime;
-scroller = uicontrol( ...
- 'Parent', f, ...
- 'Units', 'normalized', ...
- 'Position', [.95 0 .05 1], ...
- 'Callback', @scrollSummaryAxes, ...
- 'Min', 1, ...
- 'Max', max(2, nLines), ...
- 'Value', nLines, ...
- 'Style', 'slider', ...
- 'SliderStep', [1 2], ...
- 'UserData', scrollerData);
-scrollSummaryAxes(scroller, []);
-
-
-% Summarize a distribuition of data with a struct of stats.
-function summary = summarizeData(data)
-finiteData = data(isfinite(data));
-summary.min = min(finiteData);
-summary.mean = mean(finiteData);
-summary.max = max(finiteData);
-
-
-% Scroll summary axes together.
-function scrollSummaryAxes(object, event)
-scrollerData = get(object, 'UserData');
-topLine = get(object, 'Value');
-yLimit = topLine + [-scrollerData.nLinesAtATime 1];
-set(scrollerData.axes, 'YLim', yLimit);
diff --git a/Test/Interactive/rtbTestAllExampleScenes.m b/Test/Interactive/rtbRunEpicTest.m
similarity index 93%
rename from Test/Interactive/rtbTestAllExampleScenes.m
rename to Test/Interactive/rtbRunEpicTest.m
index a8329911f629f2589a9d48213a234e1daa6c3194..68ba5ea8997032c0ed9af5247eb92f5e11dadcc3 100644
--- a/Test/Interactive/rtbTestAllExampleScenes.m
+++ b/Test/Interactive/rtbRunEpicTest.m
@@ -1,7 +1,7 @@
-function results = rtbTestAllExampleScenes(varargin)
+function results = rtbRunEpicTest(varargin)
%% Run all "rtbMake..." scripts in the ExampleScenes/ folder.
%
-% results = rtbTestAllExampleScenes() renders example scenes by invoking
+% results = rtbRunEpicTest() renders example scenes by invoking
% all of the "rtbMake..." executive sripts found within the ExampleScenes/
% folder
%
@@ -19,11 +19,11 @@ function results = rtbTestAllExampleScenes(varargin)
% have a name that that includes the name of this m-file, plus the date and
% time.
%
-% rtbTestAllExampleScenes( ... 'outputRoot', outputRoot) specifies the
+% rtbRunEpicTest( ... 'outputRoot', outputRoot) specifies the
% working folder where to put rendering outputs. The default is from
% rtbDefaultHints().
%
-% rtbTestAllExampleScenes( ... 'makeFunctions', makeFunctions) specifies a
+% rtbRunEpicTest( ... 'makeFunctions', makeFunctions) specifies a
% cell array of executive functions to run. The default is to search the
% ExampleScenes/ folder for m-files that begin with "rtbMake".
%
diff --git a/Test/Interactive/rtbTestInstallation.m b/Test/Interactive/rtbTestInstallation.m
index 8243d8cb4ce61a41215a4b612f15c8a82ae795bf..ada27c2e7035d2605d8dd2a6ea5cb2ae05658f4a 100644
--- a/Test/Interactive/rtbTestInstallation.m
+++ b/Test/Interactive/rtbTestInstallation.m
@@ -77,7 +77,7 @@ end
if doAll
fprintf('\nTesting rendering with all example scripts.\n');
fprintf('This might take a while.\n');
- renderResults = rtbTestAllExampleScenes([], []);
+ renderResults = rtbRunEpicTest([], []);
else
testScenes = { ...
@@ -88,7 +88,7 @@ else
fprintf('\nTesting rendering with %d example scripts.\n', numel(testScenes));
fprintf('You should see several figures with rendered images.\n\n');
- renderResults = rtbTestAllExampleScenes('makeFunctions', testScenes);
+ renderResults = rtbRunEpicTest('makeFunctions', testScenes);
end
@@ -103,7 +103,7 @@ if ~isempty(referenceRoot)
fprintf('\nComparing local renderings\n %s\n', localRoot);
fprintf('with reference renderings\n %s\n', referenceRoot);
fprintf('You should see several more figures.\n\n');
- comparison = rtbCompareAllExampleScenes(localRoot, referenceRoot, '', 2);
+ comparison = rtbRunEpicComparison(localRoot, referenceRoot, '', 2);
else
fprintf('\nNo referenceRoot provided. Local renderings\n');
fprintf('will not be compared with reference renderings.\n');
diff --git a/Test/Interactive/rtbCheckNativeDependencies.m b/rtbCheckNativeDependencies.m
similarity index 100%
rename from Test/Interactive/rtbCheckNativeDependencies.m
rename to rtbCheckNativeDependencies.m