/**
A variant of the unix 'cut' program, with the ability to reorder fields.

tsv-select is a variation on the Unix 'cut' utility, with the added ability to reorder
fields. Lines are read from files or standard input and split on a delimiter character.
Fields are written to standard output in the order listed. Fields can be listed more
than once, and fields not listed can be written out as a group.

This program is intended both as a useful utility and a D programming language example.
Functionality and constructs used include command line argument processing, file I/O,
exception handling, ranges, tuples and strings, universal function call syntax (UFCS),
lambdas and functional programming constructs. Comments are more verbose than typical
to shed light on D programming constructs, but not to the level of a tutorial.

Copyright (c) 2015-2016, eBay Software Foundation
Initially written by Jon Degenhardt

License: Boost Licence 1.0 (http://boost.org/LICENSE_1_0.txt)
*/

// Module name defaults to file name, but hyphens not allowed, so set it here.
module tsv_select;

// Imports used by multiple routines. Others imports made in local context.
import std.stdio;
import std.typecons : tuple, Tuple;

// 'Heredoc' style help text. When printed it is followed by a getopt formatted option list.
auto helpText = q"EOS
Synopsis: tsv-select -f n[,n...] [options] [file...]

tsv-select reads files or standard input and writes specified fields to
standard output in the order listed. Similar to 'cut' with the ability to
reorder fields. Fields can be listed more than once, and fields not
listed can be output using the --rest option. Examples:

   tsv-select -f 4,2,9 file1.tsv file2.tsv
   tsv-select --delimiter ' ' -f 2,4,6 --rest last file1.txt
   cat file*.tsv | tsv-select -f 3,2,1

Options:
EOS";

/** 
Container for command line options. 
 */
struct TsvSelectOptions {
    enum RestOptionVal { none, first, last };  // Values allowed in --rest option.
    
    char delim = '\t';
    size_t[] fields;
    RestOptionVal rest;

    /* Returns a tuple. First value is true if command line arguments were successfully
     * processed and execution should continue, or false if an error occurred or the user
     * asked for help. If false, the second value is the appropriate exit code (0 or 1).
     *
     * Returning true (execution continues) means args have been validated and derived
     * values calculated. In addition, field indices have been converted to zero-based.
     */ 
    auto processArgs (ref string[] cmdArgs) {
        import std.algorithm : any, each;
        import std.getopt;
        
        try {
            arraySep = ",";    // Use comma to separate values in command line options
            auto r = getopt(
                cmdArgs,
                "f|fields",    "n[,n...]         (Required) Fields to extract. Fields are output in the order listed.", &fields,
                "r|rest",      "none|first|last  Location for remaining fields. Default: none", &rest,
                "d|delimiter", "CHR              Character to use as field delimiter. Default: TAB. (Single byte UTF-8 characters only.)", &delim
                );
            
            if (r.helpWanted) {
                defaultGetoptPrinter(helpText, r.options);
                return tuple(false, 0);
            }
        
            /* Consistency checks */
            if (fields.length == 0) {
                throw new Exception("Required option --f|fields was not supplied.");
            }
        
            if (fields.length > 0 && fields.any!(x => x == 0)) {
                throw new Exception("Zero is not a valid field number (--f|fields).");
            }

            /* Derivations */
            fields.each!((ref x) => --x);  // Convert to 1-based indexing. Using 'ref' in the lambda allows the actual
                                           // field value to be modified. Otherwise a copy would be passed.
            
        } catch (Exception exc) {
            stderr.writeln("Error processing command line arguments: ", exc.msg);
            return tuple(false, 1);
        }
        return tuple(true, 0);
    }
}

/**
Main program.
 */
int main(string[] cmdArgs) {
    TsvSelectOptions cmdopt;
    auto r = cmdopt.processArgs(cmdArgs);
    if (!r[0]) {
        return r[1];
    }
    try {
        /* Option args are removed by command line processing (getopt). The program name
         * and any files remain. Pass the files to tsvSelect.
         */
        tsvSelect(cmdopt, cmdArgs[1..$]);
    }
    catch (Exception exc) {
        stderr.writeln("Error: ", exc.msg);
        return 1;
    }

    return 0;
}

/**
tsvSelect does the primary work of the tsv-select program.
 
Input is read line by line, extracting the listed fields and writing them
out in the order specified. An exception is thrown on error.
 */
void tsvSelect(in TsvSelectOptions cmdopt, in string[] inputFiles) {
    import tsvutil: InputFieldReordering;
    import std.algorithm: splitter;
    import std.format: format;
    import std.range;

    /* InputFieldReordering copies select fields from an input line to a new buffer.
     * The buffer is reordered in the process.
     */
    auto fieldReordering = new InputFieldReordering!char(cmdopt.fields);

    /* Fields not on the --fields list are added to a separate buffer so they can be
     * output as a group (the --rest option). This is done using an 'Appender', which
     * is faster than the ~= operator. The Appender is passed a GC allocated buffer
     * that grows as needed and is reused for each line. Typically it'll grow only
     * on the first line.
     */
    bool keepingLeftOverFields = cmdopt.rest != TsvSelectOptions.RestOptionVal.none;
    auto leftOverFieldsAppender = appender!(char[][]); 

    /* Read each input file (or stdin) and iterate over each line. A filename of "-" is
     * interpreted as stdin, common behavior for unix command line tools.
     */
    foreach (filename; (inputFiles.length > 0) ? inputFiles : ["-"]) {
        auto inputStream = (filename == "-") ? stdin : filename.File();
        foreach (lineNum, line; inputStream.byLine.enumerate(1)) {
            leftOverFieldsAppender.clear;
            fieldReordering.initNewLine;
            foreach (fieldIndex, fieldValue; line.splitter(cmdopt.delim).enumerate) {
                auto numMatched = fieldReordering.processNextField(fieldIndex, fieldValue);
                if (numMatched == 0) {
                    if (keepingLeftOverFields) {
                        leftOverFieldsAppender.put(fieldValue);
                    }
                }
                else if (fieldReordering.allFieldsFilled && !keepingLeftOverFields) {
                    break;
                }
            }
            // Finished with all fields in the line.
            if (!fieldReordering.allFieldsFilled) {
                throw new Exception(
                    format("Not enough fields in line. File: %s,  Line: %s",
                           (filename == "-") ? "Standard Input" : filename, lineNum));
            }

            // Write the re-ordered line. The prefix/suffix setup is needed for chain's api.
            auto prefix = (cmdopt.rest == TsvSelectOptions.RestOptionVal.first) ? leftOverFieldsAppender.data : [];
            auto suffix = (cmdopt.rest == TsvSelectOptions.RestOptionVal.last) ? leftOverFieldsAppender.data : [];
            chain(prefix, fieldReordering.outputFields, suffix).join(cmdopt.delim).writeln;
        }
    }
}