1 /**
2 This tool runs diff comparisons of command line test output files.
3 
4 This tool supports a common TSV Utilities testing paradigm: Running command line
5 tests on the built executables to produce a set of outputs from an existing set
6 of tests. Test results are written to a files in a directory and compared to a
7 "gold" set of outputs known to be correct.
8 
9 This tool runs a version of directory diff support multiple correct output versions.
10 This is to handle the case where different compiler/library versions have different
11 valid outputs. The most common case is changes to error message text.
12 
13 Copyright (c) 2018-2020, eBay Inc.
14 Initially written by Jon Degenhardt
15 
16 License: Boost License 1.0 (http://boost.org/LICENSE_1_0.txt)
17 
18 */
19 module buildtools.diff_test_result_dirs;
20 
21 import std.conv : to;
22 import std.range;
23 import std.stdio;
24 import std.file;
25 import std.typecons : tuple;
26 import std.process : ProcessException;
27 
28 version(unittest)
29 {
30     // When running unit tests, use main from -main compiler switch.
31 }
32 else
33 {
34     int main(string[] cmdArgs)
35     {
36         /* When running in DMD code coverage mode, turn on report merging. */
37         version(D_Coverage) version(DigitalMars)
38         {
39             import core.runtime : dmd_coverSetMerge;
40             dmd_coverSetMerge(true);
41         }
42 
43         DiffOptions cmdopt;
44         auto r = cmdopt.processArgs(cmdArgs);
45         if (!r[0]) return r[1];
46 
47         int result = 0;
48         try result = diffTestResultDirs(cmdopt);
49         catch (Exception e)
50         {
51             stderr.writefln("Error [%s]: %s", cmdopt.programName, e.msg);
52             result = 1;
53         }
54         return result;
55     }
56 }
57 
58 auto helpText = q"EOS
59 Synopsis: diff-test-result-files [options] <test-dir>
60 
61 This tool runs diff comparisons of test output files. It is a modified form of
62 directory diff, comparing output files from a test run to output files from a known
63 good outputs (a "gold" set). The primary difference from a normal directory diff is
64 that the gold set may include multiple versions of a result file. A test output file
65 is considered correct if it matches any of the variants. Variants are specified in a
66 JSON config file and are used to support compiler multiple versions.
67 
68 This tool was developed for TSV Utilities tests and the default arguments support
69 this. The one required argument is <test-dir> and is usually 'latest_debug' or
70 'latest_release' corresponding to the types of test runs used in TSV Utilities tests.
71 
72 The exit status provides the result status. Zero indicates success (no differences),
73 one indicates failure (differences).
74 
75 Options:
76 EOS";
77 
78 auto helpTextVerbose = q"EOS
79 Synopsis: diff-test-result-dirs [options] <test-dir>
80 
81 This tool runs diff comparisons of test result files. The exit status gives the diff
82 status. Zero indicates success (no differences), one indicates failure (differences).
83 
84 This tool was developed for TSV Utilities command line tests. Command line tests work
85 by running a tool against a set of command line test inputs. Results are written to
86 files and compared to a "gold" set of correct results. Tests generate one or more
87 output files; all written to a single directory. The resulting comparison is a "test"
88 directory vs a "gold" directory.
89 
90 A directory level 'diff' is sufficient in many cases. This is the default behavior of
91 this tool. In some cases the corrent results depend on the compiler version. The main
92 case is error tests, where the output message may differ between runtime library
93 versions. (TSV Utilities often include exception message text in error output.)
94 
95 The latter case is what this tool was developed for. It allows for multiple versions
96 of an output file in the gold set. The files used in the test are read from a JSON
97 config file. The config file also contains the set of version files available. The
98 effect is to run a modified form of directory diff, comparing the "test" directory
99 against the "gold" directory, allowing for the presence of version files.
100 
101 The presence of the config file triggers the version-aware diff. A plain directory
102 diff is run if there is no config file.
103 
104 An example JSON config file is shown below. Each output file has an entry with one
105 required element, "name", and one optional element, "versions". If present,
106 "versions" contains a list of alternate test files.
107 
108     ==== test-config.json ====
109     {
110         "output_files" : [
111             {
112                 "name" : "test_1.txt"
113             },
114             {
115                 "name" : "test_2.txt"
116             },
117             {
118                 "name" : "test_3.txt",
119                 "versions" : [
120                     "test_3.2081.txt",
121                     "test_3.2079.txt"
122                 ]
123             }
124         ]
125     }
126 
127 Options:
128 EOS";
129 
130 struct DiffOptions
131 {
132     string programName;
133     string testDir;                            // Required argument
134     bool helpVerbose = false;                  // --help-verbose
135     string rootDir = "";                       // --r|root-dir
136     string configFile = "test-config.json";    // --c|config-file
137     string goldDir = "gold";                   // --g|gold-dir
138     bool quiet = false;                        // --q|quiet
139     size_t maxDiffLines = 40;                  // --n|max-diff-lines
140     string diffProg = "diff";                  // --diff-prog
141 
142     /* Returns a tuple. First value is true if command line arguments were successfully
143      * processed and execution should continue, or false if an error occurred or the user
144      * asked for help. If false, the second value is the appropriate exit code (0 or 1).
145      *
146      * Returning true (execution continues) means args have been validated and derived
147      * values calculated. In addition, field indices have been converted to zero-based.
148      * If the whole line is the key, the individual fields list will be cleared.
149      */
150     auto processArgs (ref string[] cmdArgs)
151     {
152         import std.getopt;
153         import std.path : baseName, stripExtension;
154 
155         programName = (cmdArgs.length > 0) ? cmdArgs[0].stripExtension.baseName : "Unknown_program_name";
156 
157         try
158         {
159             auto r = getopt(
160                 cmdArgs,
161                 "help-verbose",      "       Print full help.", &helpVerbose,
162                 "d|root-dir",        " DIR   Root directory for tests. Default: current directory", &rootDir,
163                 "c|config-file",     " FILE  Config file name. A directory diff is done if the config file doesn't exist. Default: test-config.json", &configFile,
164                 "g|gold-dir",        " DIR   Gold directory name. Default: gold", &goldDir,
165                 "q|quiet",           "       Print only the exit status, no diff output.", &quiet,
166                 "n|max-diff-lines",  " NUM   Number of diff lines to display. Zero means display all. Default: 40.", &maxDiffLines,
167                 "diff-prog",         " STR   Diff program to use. Default: diff", &diffProg,
168                 );
169 
170             if (r.helpWanted)
171             {
172                 defaultGetoptPrinter(helpText, r.options);
173                 return tuple(false, 0);
174             }
175             else if (helpVerbose)
176             {
177                 defaultGetoptPrinter(helpTextVerbose, r.options);
178                 return tuple(false, 0);
179             }
180 
181             /* Get the test directory. Should be the one command line arg remaining. */
182             if (cmdArgs.length == 2)
183             {
184                 testDir = cmdArgs[1];
185             }
186             else if (cmdArgs.length < 2)
187             {
188                 throw new Exception("A test directory is required.");
189             }
190             else
191             {
192                 throw new Exception("Unexpected arguments.");
193             }
194         }
195         catch (Exception exc)
196         {
197             stderr.writefln("[%s] Error processing command line arguments: %s", programName, exc.msg);
198             return tuple(false, 1);
199         }
200         return tuple(true, 0);
201     }
202 }
203 
204 int diffTestResultDirs(DiffOptions cmdopt)
205 {
206     import std.json;
207     import std.file : dirEntries, readText;
208     import std.path : absolutePath, baseName, buildNormalizedPath;
209     import std.process : escapeShellCommand, executeShell;
210     import std.string : KeepTerminator, lineSplitter;
211     import std.format : format;
212     import std.algorithm : any;
213     import std.array : join;
214 
215     int testResultStatus = 0;
216     string diffOutput = "";
217 
218     auto configFilePath = (cmdopt.rootDir.length == 0)
219         ? cmdopt.configFile.absolutePath.buildNormalizedPath
220         : [cmdopt.rootDir, cmdopt.configFile].buildNormalizedPath.absolutePath;
221 
222     auto testDirPath = (cmdopt.rootDir.length == 0)
223         ? cmdopt.testDir.absolutePath.buildNormalizedPath
224         : [cmdopt.rootDir, cmdopt.testDir].buildNormalizedPath.absolutePath;
225 
226     auto goldDirPath = (cmdopt.rootDir.length == 0)
227         ? cmdopt.goldDir.absolutePath.buildNormalizedPath
228         : [cmdopt.rootDir, cmdopt.goldDir].buildNormalizedPath.absolutePath;
229 
230     auto useDirectoryDiff = !configFilePath.exists;
231 
232     if (!testDirPath.exists || !goldDirPath.exists || !testDirPath.isDir || !goldDirPath.isDir)
233     {
234         testResultStatus = 1;
235         if (!cmdopt.quiet)
236         {
237             if (!testDirPath.exists) diffOutput ~= format("Test directory not found: '%s'\n", testDirPath);
238             else if (!testDirPath.isDir) diffOutput ~= format("Test directory not a directory: '%s'\n", testDirPath);
239 
240             if (!goldDirPath.exists) diffOutput ~= format("Gold directory not found: '%s'\n", goldDirPath);
241             else if (!goldDirPath.isDir) diffOutput ~= format("Gold directory not a directory: '%s'\n", goldDirPath);
242         }
243     }
244     else if (useDirectoryDiff)
245     {
246         auto diffCmdArgs = [cmdopt.diffProg, testDirPath, goldDirPath];
247         auto diffResult = diffCmdArgs.escapeShellCommand.executeShell;
248         testResultStatus = diffResult.status;
249         if (diffResult.status != 0 && !cmdopt.quiet) diffOutput ~= diffResult.output;
250     }
251     else
252     {
253         /* These AAs keep the test output file names found in the config file. At the
254          * of end of processing the test and gold directories are walked to see that
255          * every file in the directory is accounted for by the config file.
256          */
257         bool[string] outputFileNames;
258         bool[string] versionFileNames;
259 
260         JSONValue configData;
261         try configData = configFilePath.readText.parseJSON;
262         catch (Exception e) throw new Exception(format("Could not processing config file '%s': %s", configFilePath, e.msg));
263 
264         foreach (outputFileJSON; configData["output_files"].array)
265         {
266             int fileStatus = 0;
267             auto outputFileName = outputFileJSON["name"].str;
268             auto testFilePath = buildNormalizedPath(testDirPath, outputFileName);
269             auto goldFilePath = buildNormalizedPath(goldDirPath, outputFileName);
270 
271             if (!testFilePath.exists || !goldFilePath.exists)
272             {
273                 fileStatus = 1;
274                 if (!cmdopt.quiet)
275                 {
276                     diffOutput ~= format("->>> Comparsion failed for config entry: '%s'\n", outputFileName);
277                     if (!testFilePath.exists) diffOutput ~= format("  Test file not found: '%s'\n", testFilePath);
278                     if (!goldFilePath.exists) diffOutput ~= format("  Gold file not found: '%s'\n", goldFilePath);
279                 }
280             }
281             else
282             {
283                 bool fileMatch = true;
284                 auto diffCmdArgs = [cmdopt.diffProg, testFilePath, goldFilePath];
285                 auto diffResult = diffCmdArgs.escapeShellCommand.executeShell;
286 
287                 if (diffResult.status != 0)
288                 {
289                     fileMatch = false;
290                     if ("versions" in outputFileJSON)
291                     {
292                         bool versionFileDiff(JSONValue versionFileNameJSON)
293                         {
294                             auto versionFileName = versionFileNameJSON.str;
295                             auto versionFilePath = buildNormalizedPath(goldDirPath, versionFileName);
296                             auto versionDiffCmdArgs = [cmdopt.diffProg, testFilePath, versionFilePath];
297                             auto versionDiffResult = versionDiffCmdArgs.escapeShellCommand.executeShell;
298                             return (versionDiffResult.status == 0);
299                         }
300 
301                         auto versionFiles = outputFileJSON["versions"].array;
302                         fileMatch = versionFiles.any!versionFileDiff;
303                     }
304                 }
305 
306                 if (!fileMatch)
307                 {
308                     fileStatus = diffResult.status;
309                     if (!cmdopt.quiet)
310                     {
311                         diffOutput ~= format("->>> Diff failed for config entry: '%s'", outputFileName);
312                         auto numVersions = ("versions" !in outputFileJSON) ? 0 : outputFileJSON["versions"].array.length;
313                         if (numVersions > 0) diffOutput ~= format(" (including %d alternate version files)", numVersions);
314                         diffOutput ~= "\n\n";
315                         diffOutput ~= format("%s %s %s\n", cmdopt.diffProg, testFilePath, goldFilePath);
316                         diffOutput ~= diffResult.output;
317                         diffOutput ~= "\n";
318                     }
319                 }
320             }
321             if (testResultStatus == 0 && fileStatus != 0) testResultStatus = fileStatus;
322         }
323 
324         /* Add confile entries to the list of AAs of file names. Also check that version
325          * files exist in the gold directory. The base entry has already been checked.
326          */
327         foreach (outputFileJSON; configData["output_files"].array)
328         {
329             auto outputFileName = outputFileJSON["name"].str;
330             outputFileNames[outputFileName] = true;
331             if ("versions" in outputFileJSON)
332             {
333                 foreach (versionFileJSON; outputFileJSON["versions"].array)
334                 {
335                     auto versionFileName = versionFileJSON.str;
336                     versionFileNames[versionFileName] = true;
337                     auto versionFilePath = buildNormalizedPath(goldDirPath, versionFileName);
338                     if (!versionFilePath.exists)
339                     {
340                         if (testResultStatus == 0) testResultStatus = 1;
341                         if (!cmdopt.quiet) diffOutput ~= format("->>> Invalid config entry '%s', version file does not exist: '%s'\n", outputFileName, versionFilePath);
342                     }
343                 }
344             }
345         }
346 
347         /* Check that all files in test and gold directories are in the config file. */
348         foreach (string filePath; dirEntries(testDirPath, SpanMode.shallow))
349         {
350             auto fileName = filePath.baseName;
351             if (fileName !in outputFileNames)
352             {
353                 if (testResultStatus == 0) testResultStatus = 1;
354                 if (!cmdopt.quiet) diffOutput ~= format("->>> Test directory file not referenced in config: '%s'\n", filePath);
355             }
356         }
357 
358         foreach (string filePath; dirEntries(goldDirPath, SpanMode.shallow))
359         {
360             auto fileName = filePath.baseName;
361             if (fileName !in outputFileNames && fileName !in versionFileNames)
362             {
363                 if (testResultStatus == 0) testResultStatus = 1;
364                 if (!cmdopt.quiet) diffOutput ~= format("->>> Gold directory file not referenced in config: '%s'\n", filePath);
365             }
366         }
367     }
368 
369     if (testResultStatus != 0 && !cmdopt.quiet)
370     {
371         write(
372             (cmdopt.maxDiffLines == 0)
373             ? diffOutput
374             : diffOutput.lineSplitter!(Yes.keepTerminator).take(cmdopt.maxDiffLines).join
375             );
376     }
377 
378     return testResultStatus;
379 }