/**
 * TabDataReader -- Translates data sets containing ArrayParam objects to and from tab
 * delimited arrays.
 *
 * Copyright (C) 2003-2017, by Joseph A. Huwaldt. All rights reserved.
 *
 * This library is free software; you can redistribute it and/or modify it under the terms
 * of the GNU Lesser General Public License as published by the Free Software Foundation;
 * either version 2 of the License, or (at your option) any later version.
 *
 * This library is distributed in the hope that it will be useful, but WITHOUT ANY
 * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A
 * PARTICULAR PURPOSE. See the GNU Library General Public License for more details.
 *
 * You should have received a copy of the GNU Lesser General Public License along with
 * this program; if not, write to the Free Software Foundation, Inc., 59 Temple Place -
 * Suite 330, Boston, MA 02111-1307, USA. Or visit: http://www.gnu.org/licenses/lgpl.html
 */
package jahuwaldt.js.datareader;

import jahuwaldt.io.FileUtils;
import jahuwaldt.js.util.TextTokenizer;
import jahuwaldt.swing.AppUtilities;
import java.awt.Frame;
import java.io.*;
import java.text.MessageFormat;
import java.util.List;
import java.util.NoSuchElementException;
import javax.measure.quantity.Dimensionless;
import javax.measure.unit.Unit;
import javax.measure.unit.UnitFormat;
import javolution.text.Text;
import javolution.text.TypeFormat;
import javolution.util.FastTable;

/**
 * This class translates data between a list of DataSet data structures and a stream
 * containing tab, space, or comma delimited columns of array data with a single line
 * header indicating the names of the parameters and an optional second line indicating
 * the units of each parameter. Only arrays can be read or written to this format. They
 * all must be the same length (in the same case).
 *
 * <p> Modified by: Joseph A. Huwaldt </p>
 *
 * @author Joseph A. Huwaldt, Date: May 19, 2003
 * @version March 21, 2017
 */
public class TabDataReader implements DataReader {

    //  The default name given to DAT data sets.
    private CharSequence _defaultSetName = RESOURCES.getString("defSetName");

    //  The delimiters to use to separate columns in the file.
    //  Use space, tab, and comma as delimiters.
    private String DELIMITERS = " \t,";

    //  The delimiter to use when writing out files.
    private static final String DELIM = "\t";

    //  Text displayed when there are no units defined for an item.
    public static final String NO_UNITS = "nd";

    //  A brief description of the data read by this reader.
    private static final String DESCRIPTION = RESOURCES.getString("tabDesc");

    //  The preferred file extension for files of this reader's type.
    public static final String EXTENSION = RESOURCES.getString("tabDefExt");

    /**
     * Returns a string representation of the object. This will return a brief description
     * of the format read by this reader.
     */
    @Override
    public String toString() {
        return DESCRIPTION;
    }

    /**
     * Returns the preferred file extension (not including the ".") for files of this
     * DataReader's type.
     *
     * @return The preferred file extension for this file's type.
     */
    @Override
    public String getExtension() {
        return EXTENSION;
    }

    /**
     * Compares this object with the specified object for order based on the
     * <code>toString().compareTo(o.toString())</code> method. Returns a negative integer,
     * zero, or a positive integer as this object is less than, equal to, or greater than
     * the specified object.
     */
    @Override
    public int compareTo(DataReader o) {
        return this.toString().compareTo(o.toString());
    }

    /**
     * Method that determines if this reader can read data from the specified input
     * stream.
     *
     * @param pathName The path to the file to be read.
     * @param input    An input stream containing the data to be read. Any methods that
     *                 read from this stream must first set a mark and then reset back to
     *                 that mark before the method returns (even if it returns with an
     *                 exception).
     * @return DataReader.NO if the file is not recognized at all or DataReader.MAYBE if
     * the file has an appropriate extension.
     * @throws java.io.IOException If the input stream could not be read.
     */
    @Override
    public int canReadData(String pathName, BufferedInputStream input) throws IOException {

        //  Get the file name extension.
        String extension = FileUtils.getExtension(pathName);

        // Get the list of extensions that we MIGHT be able to read.
        String[] extensions = RESOURCES.getString("tabExtensions").split(",");
        for (String ext : extensions) {
            if (extension.equalsIgnoreCase(ext.trim()))
                return MAYBE;
        }

        return NO;
    }

    /**
     * Returns true. This class can write data to a tabbed arrays formatted file.
     * 
     * @return Always returns true.
     */
    @Override
    public boolean canWriteData() {
        return true;
    }

    /**
     * The Tabbed Array format can save only a single case of arrays.
     *
     * @param parent Determines the Frame in which the dialog is displayed; if null, or if
     *               the parentComponent has no Frame, a default Frame is used.
     * @param data   The input data set that is to be selected for saving.
     * @return A list of DataSet objects containing the selected data to be saved. Could
     *         return null if the user selects nothing.
     */
    @Override
    public List<DataSet> selectDataForSaving(Frame parent, List<DataSet> data) {

        if (data == null || data.size() < 1)
            return null;

        DataSet selected;

        if (data.size() == 1 && data.get(0).size() == 1)
            //  If there is just one case, save that.
            selected = data.get(0);

        else {
            //  If there is more than one case, 
            //  ask the user to select cases to export.
            SelectCasesDialog dialog = new SelectCasesDialog(parent, RESOURCES.getString("selectCaseTitle"),
                    RESOURCES.getString("selectCaseMsg"), data, true, true);
            dialog.setLocation(AppUtilities.dialogPosition(dialog));
            dialog.setVisible(true);

            //  Retrieve the selected case.
            selected = dialog.getSelected();
            dialog.dispose();
        }
        if (selected == null || selected.size() < 1)
            return null;

        FastTable<DataSet> output = FastTable.newInstance();
        output.add(selected);

        return output;
    }

    /**
     * Method that reads in tab, space or comma delimited array data from the specified
     * input stream and returns that data as a list of {@link DataSet} objects.
     *
     * @param pathName The path to the file to be read.
     * @param input    An input stream containing the tab, space or comma delimited array
     *                 data.
     * @return A list of DataSet objects that contains the data read in from the
     * specified stream (will contain a single data set which will contain a single case
     * with ArrayParam objects for each column).
     * @throws IOException If there is a problem reading the specified stream.
     */
    @Override
    public List<DataSet> read(String pathName, InputStream input) throws IOException {

        //  Wrap the input stream in a line number reader.
        LineNumberReader reader = new LineNumberReader(new InputStreamReader(input));

        //  Create an empty list to store DataSets into.
        FastTable<DataSet> dataSets = FastTable.newInstance();

        //  Tab files contain only a single data set, so create that.
        DataSet set = DataSet.newInstance(_defaultSetName);

        //  Tab files contain only a single case, so create that.
        DataCase aCase = DataCase.newInstance(_defaultSetName);

        //  Parse out the parameter names from the 1st line.
        FastTable<Text> names = inputParameterNames(reader);

        //  Read in the optional units from the 2nd line.
        FastTable<Unit<?>> units = inputUnits(reader, names.size());

        //  Now read in the arrays of data.
        inputDataArrays(reader, aCase, names, units);

        //  Add the case to the set.
        set.add(aCase);

        //  Add the set to the list of data sets being output.
        dataSets.add(set);

        //  Clean up before leaving.
        FastTable.recycle(names);
        FastTable.recycle(units);

        return dataSets;
    }

    /**
     * Method that writes out all the data stored in the specified list of {@link DataSet}
     * objects to the specified output stream in tabbed array format. Only the 1st case in
     * the 1st data set in the specified list is written out since the tabbed array format
     * does not support multiple cases. Only arrays are written out since the tabbed array
     * format does not support scalars or text notes.
     *
     * @param output The output stream to which the data is to be written.
     * @param data   A list of {@link DataSet} objects containing data to be written out.
     *               Only the 1st case in the 1st data set in the list is written out.
     * @throws IOException If there is a problem writing to the specified stream.
     */
    @Override
    public void write(OutputStream output, List<DataSet> data) throws IOException {

        if (data.size() < 1)
            return;

        //  Wrap the output stream in a writer.
        BufferedWriter writer = new BufferedWriter(new OutputStreamWriter(output));

        // Extract the 1st data set from the list.
        DataSet set = data.get(0);

        //  Extract the 1st case from the data set.
        DataCase aCase = set.get(0);

        //  Extract all the array parameters.
        List<ArrayParam> arrays = aCase.getAllArrays();
        if (arrays == null || arrays.size() < 1)
            throw new IOException(RESOURCES.getString("emptyDataSetErr"));

        //  Write out the arrays.
        writeArrays(writer, arrays);

        //  Clean up before leaving.
        FastTable.recycle((FastTable<?>)arrays);

        writer.flush();
    }

    /**
     * Sets the default set name to use.
     * 
     * @param name The name to use as the default set name.
     */
    @Override
    public void setDefaultSetName(CharSequence name) {
        _defaultSetName = name;
    }

    /**
     * Method that reads in and parses out the optional parameter names from the 1st line
     * of the file. If the file does not contain a header line, then a list of generic
     * parameter names is returned.
     */
    private FastTable<Text> inputParameterNames(LineNumberReader reader) throws IOException {

        //  Read in the 1st line.
        reader.mark(65536);
        String line = reader.readLine();
        if (line == null)
            throw new IOException(MessageFormat.format(
                    RESOURCES.getString("eolErr"),reader.getLineNumber()));

        //  Create a list of accumulated data from the 1st line.
        FastTable<Text> columns = FastTable.newInstance();

        //  Create a list to contain the parameter names.
        FastTable<Text> pNames = FastTable.newInstance();

        //  Determine the delimiter(s) to use.
        if (line.contains("\t"))
            DELIMITERS = "\t";
        else
            DELIMITERS = " ,\t";

        //  Create a tokenizer.
        TextTokenizer tokenizer = TextTokenizer.valueOf(line, DELIMITERS);
        tokenizer.setHonorQuotes(true);
        tokenizer.setQuoteChar('"');

        //  Add each label found to the list of parameter names.
        boolean foundText = false;
        int i = 1;
        for (Text token : tokenizer) {
            token = token.replace("\"", " ").trim();    //  Trim off any quote characters.
            columns.add(token);
            try {
                TypeFormat.parseDouble(token);

                //  Conversion to double did not fail, we have a number -- use default label.
                pNames.add(Text.valueOf(MessageFormat.format(RESOURCES.getString("defColumnLabel"),i)));

            } catch (NumberFormatException e) {
                //  Conversion to double failed, we have found text.
                foundText = true;
                pNames.add(token);
            }
            ++i;
        }

        TextTokenizer.recycle(tokenizer);
        FastTable.recycle(columns);

        int length = pNames.size();
        if (length < 1)
            throw new IOException(RESOURCES.getString("noParamNamesErr"));

        //  If no labels were found, then reset the read buffer to read the 1st lime again.
        if (!foundText)
            reader.reset();

        return pNames;
    }

    /**
     * Reads in the optional units (if they are present) and returns them in a list. The
     * specified number of parameter units are returned.
     */
    private FastTable<Unit<?>> inputUnits(LineNumberReader reader, int numParams) throws IOException {

        //  Create a list of "no units" units.
        FastTable<Unit<?>> units = FastTable.newInstance();
        for (int i = 0; i < numParams; ++i)
            units.add(Dimensionless.UNIT);
        int i = -1;

        //  Mark position and read in unit line.
        reader.mark(65536);
        String line = reader.readLine();

        //  Create a tokenizer.
        TextTokenizer tokenizer = TextTokenizer.valueOf(line, DELIMITERS);

        Text unitName = null;
        try {
            //  Loop through the line.
            for (i = 0; i < numParams; ++i) {
                unitName = Text.valueOf(NO_UNITS);

                if (tokenizer.hasMoreTokens()) {
                    unitName = tokenizer.nextToken().trim();

                    //  Is this "unit" actually a number?
                    if (i == 0) {
                        try {
                            TypeFormat.parseDouble(unitName);

                            //  If that didn't cause an exception, stop processing.
                            //  We have found numbers, not units.
                            reader.reset();
                            break;
                        } catch (NumberFormatException e) {
                        }
                    }
                }

                if (!unitName.equals(Text.valueOf(NO_UNITS))) {
                    //  Do some common unit name substitutions.
                    unitName = unitName.replace("sec", "s");
                    unitName = unitName.replace("-", "*");
                    unitName = unitName.replace("deg", "\u00B0");
                    Unit<?> unit = Unit.valueOf(unitName);
                    units.set(i, unit);
                }
            }

        } catch (IllegalArgumentException e) {
            ++i;
            throw new IOException(MessageFormat.format(
                    RESOURCES.getString("unkwnUnitErr"),unitName,i,reader.getLineNumber()));
        }

        return units;
    }

    /**
     * Method that reads in arrays of delimited data and creates array parameters out of
     * them.
     *
     * @param reader The reader for our input stream.
     * @param aCase  The data case that parameters should be added to.
     * @param names  A list of parameter names.
     * @param units  A list of parameter units.
     * @param nf     The NumberFormat used to parse the numbers.
     */
    private void inputDataArrays(LineNumberReader reader, DataCase aCase,
            List<Text> names, FastTable<Unit<?>> units) throws IOException {

        int numParams = names.size();
        int lineNum = reader.getLineNumber();

        //  Read in all the lines of the file, storing them in memory, to count how
        //  many lines (array elements) there are.
        FastTable<Text> data = FastTable.newInstance();
        String aLine;
        do {
            //  Read a line from the file.
            aLine = reader.readLine();

            //  Skip blank lines and lines starting with "--".
            if (aLine == null)
                break;
            if (aLine.startsWith("--") || aLine.trim().equals(""))
                continue;

            //  Add the line to our list of lines.
            data.add(Text.valueOf(aLine));

        } while (true);

        //  Create a list of properly sized arrays, one for each parameter.
        FastTable<double[]> arrays = FastTable.newInstance();
        int numElements = data.size();
        for (int i = 0; i < numParams; ++i) {
            arrays.add(new double[numElements]);
        }

        //  Now go through the list of data strings and parse out the numbers.
        TextTokenizer tokenizer = TextTokenizer.newInstance();
        tokenizer.setDelimiters(DELIMITERS);
        Text token = null;
        int column = 0;
        try {
            for (int row = 0; row < numElements; ++row, ++lineNum) {
                Text text = data.get(row);

                //  Create a tokenizer.
                tokenizer.setText(text);

                //  Loop through the line.
                for (column = 0; column < numParams; ++column) {
                    //  Parse out the value.
                    token = tokenizer.nextToken();
                    token = token.trim().toUpperCase();

                    //  Catch any NaNs.
                    double value = Double.NaN;
                    if (!token.contentEquals("NAN")) {

                        //  The POST2 to tabular data file converter has a bug that outputs
                        //  some numbers without the "E" exponent. This code trys to deal 
                        //  with that situation.
                        int idx = token.indexOf("E");
                        if (idx < 0) {
                            //  Look for an "embedded" + or - sign that is NOT the 1st character in the token.
                            idx = token.indexOf("-");
                            if (idx < 0)
                                idx = token.indexOf("+");

                            if (idx > 0) {
                                // Insert an "E" before the embedded + or - sign.
                                token = token.insert(idx, Text.valueOf("E"));
                            }
                        }

                        //  Now parse the token into a number.
                        value = TypeFormat.parseDouble(token);
                    }

                    //  Store the value.
                    arrays.get(column)[row] = value;
                }
            }   //  next row

        } catch (NumberFormatException e) {
            ++column;
            ++lineNum;
            String msg = "null";
            if (token != null)
                msg = token.toString();
            throw new IOException(MessageFormat.format(RESOURCES.getString("numFmtErr"),
                    msg, lineNum, column, names.get(column - 1)),e);

        } catch (NoSuchElementException e) {
            ++lineNum;
            throw new IOException(MessageFormat.format(RESOURCES.getString("eolErr"),lineNum), e);
        }

        //  Convert from simple arrays to ArrayParam instances.
        FastTable<ArrayParam> arrParams = FastTable.newInstance();
        for (int i = 0; i < numParams; ++i) {
            ArrayParam array = ArrayParam.valueOf(names.get(i), units.get(i), arrays.get(i));
            arrParams.add(array);
        }

        //  The list of arrays should now be filled with numbers.
        //  Put the arrays in the data case.
        aCase.addAll(arrParams);

        //  Clean up before leaving.
        TextTokenizer.recycle(tokenizer);
        FastTable.recycle(data);
        FastTable.recycle(arrays);
        FastTable.recycle(arrParams);
    }

    /**
     * Method that writes out the specified list of array parameter objects to a tab
     * delimited table of arrays.
     */
    private void writeArrays(BufferedWriter writer, List<ArrayParam> arrays) throws IOException {

        //  First, write out all the parameter names.
        for (ArrayParam array : arrays) {
            Text name = Text.valueOf(array.getName());
            name.print(writer);
            writer.write(DELIM);
        }
        writer.newLine();

        //  Now write out the units.
        UnitFormat fmt = UnitFormat.getUCUMInstance();
        for (ArrayParam array : arrays) {
            Unit<?> units = array.getUnit();
            if (units != null && !units.equals(Dimensionless.UNIT))
                fmt.format(units, writer);
            else
                Text.valueOf(NO_UNITS).print(writer);
            writer.write(DELIM);
        }
        writer.newLine();

        //  Start writing out the arrays of numbers.
        int numElements = arrays.get(0).size();
        for (int i = 0; i < numElements; ++i) {
            for (ArrayParam array : arrays) {
                TypeFormat.format(array.getValue(i), writer);
                writer.write(DELIM);
            }
            writer.newLine();
        }

    }

}