/*
 * JBoss, Home of Professional Open Source.
 * See the COPYRIGHT.txt file distributed with this work for information
 * regarding copyright ownership.  Some portions may be licensed
 * to Red Hat, Inc. under one or more contributor license agreements.
 *
 * This library is free software; you can redistribute it and/or
 * modify it under the terms of the GNU Lesser General Public
 * License as published by the Free Software Foundation; either
 * version 2.1 of the License, or (at your option) any later version.
 *
 * This library is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * Lesser General Public License for more details.
 *
 * You should have received a copy of the GNU Lesser General Public
 * License along with this library; if not, write to the Free Software
 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
 * 02110-1301 USA.
 * Modified by
 * @author  Data Virtuality GmbH
 * @date    $Date$
 *
 */
// https://issues.jboss.org/browse/TEIID-2859
// https://issues.jboss.org/browse/TEIID-3167
// https://issues.jboss.org/browse/TEIID-3163
// https://issues.jboss.org/browse/TEIID-2898
// https://issues.jboss.org/browse/TEIID-2896
// https://issues.jboss.org/browse/TEIID-3346
// https://issues.jboss.org/browse/TEIID-4449
// https://issues.jboss.org/browse/TEIID-2897
// https://issues.jboss.org/browse/TEIID-5370
// https://issues.redhat.com/browse/TEIID-5938
package org.teiid.query.processor.relational;

import com.datavirtuality.dv.core.interfaces.DVCaller;
import org.teiid.api.exception.query.ExpressionEvaluationException;
import org.teiid.common.buffer.BlockedException;
import org.teiid.common.buffer.BufferManager;
import org.teiid.common.buffer.TupleBatch;
import org.teiid.core.TeiidComponentException;
import org.teiid.core.TeiidProcessingException;
import org.teiid.core.TeiidRuntimeException;
import org.teiid.core.types.ClobImpl;
import org.teiid.core.types.ClobType;
import org.teiid.core.types.DataTypeManager;
import org.teiid.core.types.TransformationException;
import org.teiid.dqp.internal.process.RequestWorkItem;
import org.teiid.query.QueryPlugin;
import org.teiid.query.processor.ProcessorDataManager;
import org.teiid.query.sql.LanguageObject;
import org.teiid.query.sql.lang.TextTable;
import org.teiid.query.sql.lang.TextTable.TextColumn;
import org.teiid.query.sql.symbol.Expression;
import org.teiid.query.util.CommandContext;

import java.io.BufferedReader;
import java.io.IOException;
import java.io.Reader;
import java.nio.charset.MalformedInputException;
import java.sql.SQLException;
import java.util.*;

/**
 * Handles text file processing.
 *
 * TODO: allow for a configurable line terminator
 */
public class TextTableNode extends SubqueryAwareRelationalNode {

	private static final String MAX_WIDTH_OPT_NAME = "TEXT_TABLE_MAX_WIDTH";

	private Integer maxWidth = (Integer) DVCaller.call(63, MAX_WIDTH_OPT_NAME, null, null);

	private TextTable table;

	//initialized state
	private int skip = 0;
	private int header = -1;
	private boolean noQuote;
	private char quote;
	private char delimiter;
	private int lineWidth;
    private int[] projectionIndexes;
    private Map<String, List<String>> parentLines;

    //per file state
	private BufferedReader reader;
	private int textLine = 0;
	private Map<String, Integer> nameIndexes;
	private String systemId;
	private int rowNumber;

	private boolean cr;
	private boolean eof;

	private volatile boolean running;
	private volatile TeiidRuntimeException asynchException;

	private int limit = -1;

	private boolean noTrim;

	private char newLine = '\n';
	private boolean crNewLine = true;
	private int errorCount;
	private TeiidProcessingException initException;

	public TextTableNode(int nodeID) {
		super(nodeID);
	}

	@Override
	public void initialize(CommandContext context, BufferManager bufferManager,
			ProcessorDataManager dataMgr) {
		super.initialize(context, bufferManager, dataMgr);

		if (projectionIndexes != null) {
			return;
		}
		if (table.getSkip() != null) {
			skip = table.getSkip();
		}
		if (table.getHeader() != null) {
			skip = Math.max(table.getHeader(), skip);
			header = table.getHeader() - 1;
		}
		if (table.isFixedWidth()) {
			for (TextColumn col : table.getColumns()) {
				if (col.getWidth() != null) {
					lineWidth += col.getWidth();
				}
			}
		} else {
			if (table.getDelimiter() == null) {
				delimiter = ',';
			} else {
				delimiter = table.getDelimiter();
			}
			if (table.getQuote() == null) {
				quote = '"';
			} else {
				noQuote = table.isEscape();
				quote = table.getQuote();
			}
			for (TextColumn column : table.getColumns()) {
				if (column.getSelector() != null) {
					if (parentLines == null) {
						parentLines = new HashMap<String, List<String>>();
					}
					parentLines.put(column.getSelector(), null);
				}
	        }
			// EXTENSION
			// ISSUE: DVCORE-2607
			// FIXED BY: Roman Pedchenko
			// DATE: 2015.02.17
			// lineWidth = table.getColumns().size() * DataTypeManager.MAX_STRING_LENGTH;
			lineWidth = maxWidth != null ? maxWidth : table.getColumns().size() * DataTypeManager.getEstimatedStringLength();
			// END EXTENSION
		}
		if (table.isUsingRowDelimiter()) {
			Character c = table.getRowDelimiter();
			if (c != null) {
				this.newLine = c;
				this.crNewLine = false;
			}
		}
        Map<Expression, Integer> elementMap = createLookupMap(table.getProjectedSymbols());
        this.projectionIndexes = getProjectionIndexes(elementMap, getElements());
		try {
			DVCaller.callWithExc(137, table.getErrorLog());
		} catch (Throwable e) {
			if (e instanceof TeiidProcessingException)
				initException = (TeiidProcessingException)e;
			else
				initException = new TeiidProcessingException(e);
		}
	}

	@Override
	public void closeDirect() {
		super.closeDirect();
		reset();
	}

	@Override
	public synchronized void reset() {
		super.reset();
		if (this.reader != null) {
			try {
				this.reader.close();
			} catch (IOException e) {
			}
			this.reader = null;
		}
		this.nameIndexes = null;
		this.textLine = 0;
		this.rowNumber = 0;
		this.cr = false;
		this.eof = false;
		if (this.parentLines != null) {
			for (Map.Entry<String, List<String>> entry : this.parentLines.entrySet()) {
				entry.setValue(null);
			}
		}
		this.running = false;
		this.asynchException = null;
		this.limit = -1;
	}

	public void setTable(TextTable table) {
		this.table = table;
		this.noTrim = table.isNoTrim();
	}

	@Override
	public TextTableNode clone() {
		TextTableNode clone = new TextTableNode(getID());
		this.copyTo(clone);
		clone.setTable(table);
		return clone;
	}

	@Override
	public void open() throws TeiidComponentException, TeiidProcessingException {
		super.open();
		if (getParent() instanceof LimitNode) {
			LimitNode parent = (LimitNode)getParent();
			if (parent.getLimit() > 0) {
				limit = parent.getLimit() + parent.getOffset();
			}
		}
	}

	@Override
	protected synchronized TupleBatch nextBatchDirect() throws BlockedException,
			TeiidComponentException, TeiidProcessingException {
		if (initException != null)
			throw initException;

		if (reader == null) {
			initReader();
		}

		if (reader == null) {
			terminateBatches();
			return pullBatch();
		}

		if (isLastBatch()) {
			return pullBatch();
		}

		if (isBatchFull()) {
			TupleBatch result = pullBatch();
			processAsynch(); // read ahead
			return result;
		}

		unwrapException(asynchException);

		processAsynch();

		if (this.getContext().getWorkItem() == null) {
			//this is for compatibility with engine tests that are below the level of using the work item
			synchronized (this) {
				while (running) {
					try {
						this.wait();
					} catch (InterruptedException e) {
						throw new TeiidRuntimeException(e);
					}
				}
			}
		}

		throw BlockedException.block("Blocking on results from file processing."); //$NON-NLS-1$
	}

	private void processAsynch() {
		if (!running) {
			running = true;
			final Reader r = this.reader;
			getContext().getExecutor().execute(new Runnable() {
				@Override
				public void run() {
					try {
						process(r);
					} catch (TeiidRuntimeException e) {
						asynchException = e;
					} catch (Throwable e) {
						asynchException = new TeiidRuntimeException(e);
					} finally {
						running = false;
						RequestWorkItem workItem = TextTableNode.this.getContext().getWorkItem();
						if (workItem != null) {
							workItem.moreWork();
						} else {
							synchronized (TextTableNode.this) {
								TextTableNode.this.notifyAll();
							}
						}
					}
				}
			});
		}
	}

	private void process(Reader r) throws TeiidProcessingException {
		while (true) {
			synchronized (this) {
				if (isBatchFull() || r != this.reader) {
					return;
				}
				StringBuilder line = readLine(lineWidth, table.isFixedWidth(), false);

				if (line == null) {
					terminateBatches();
					break;
				}

				try {
					String parentSelector = null;
					if (table.getSelector() != null) {
						if (line.length() < table.getSelector().length()) {
							continue;
						}
						if (!line.substring(0, table.getSelector().length()).equals(table.getSelector())) {
							if (parentLines == null) {
								continue; //doesn't match any selector
							}
							parentSelector = line.substring(0, table.getSelector().length());

							if (!parentLines.containsKey(parentSelector)) {
								continue; //doesn't match any selector
							}
						}
					}

					List<String> vals = parseLine(line);

					if (parentSelector != null) {
						this.parentLines.put(parentSelector, vals);
						continue;
					} else if (table.getSelector() != null && !table.getSelector().equals(vals.get(0))) {
						continue;
					}

					rowNumber++;

					List<Object> tuple = new ArrayList<Object>(projectionIndexes.length);
					Map<Integer, Integer> missingOrdinalColumns = new HashMap<>(table.getColumns().size());
					int colIndex = 0;
					for (int output : projectionIndexes) {
						TextColumn col = table.getColumns().get(output);
						String val = null;
						int index = output;
						boolean missing = false;

						boolean missingColumnsInVals = table.getColumns().size() > vals.size();
						Integer addedOrdinalColumnsCount = missingOrdinalColumns.get(output);
						while (missingColumnsInVals && output >= colIndex) { // count the number of missing ordinal columns for each index up to the largest projection index
							col = table.getColumns().get(colIndex);
							addedOrdinalColumnsCount = missingOrdinalColumns.get(colIndex - 1);
							if (col.isOrdinal())
								addedOrdinalColumnsCount = addedOrdinalColumnsCount != null ? addedOrdinalColumnsCount + 1 : 1;
							missingOrdinalColumns.put(colIndex, addedOrdinalColumnsCount);
							colIndex++;
						}

						if (col.isOrdinal()) {
							tuple.add(rowNumber);
							continue;
						}

						if (col.getSelector() != null) {
							vals = this.parentLines.get(col.getSelector());
							index = col.getPosition() - 1;
						} else if (nameIndexes != null) {
							Integer headerIndex = nameIndexes.get(col.getName());
							if (headerIndex != null) {
								index = headerIndex;
							} else {
								missing = true;
							}
						} else if ((addedOrdinalColumnsCount != null) && (index > addedOrdinalColumnsCount) && missingColumnsInVals) {
							// BUG: Ordinality column in TextTable can break parsing when column order is assumed for column mapping
							// ISSUE: DVCORE-7767
							// SOLUTION: Shifted subsequent columns values to right if an ordinal is included in response
							// FIXED BY: Muhammad Imran Ansari
							// DATE: 2022.09.16
							index = index - addedOrdinalColumnsCount;
							// END BUG
						}
						if (vals == null || index >= vals.size() || missing) {
							//throw new TeiidProcessingException(QueryPlugin.Util.getString("TextTableNode.no_value", col.getName(), textLine, systemId)); //$NON-NLS-1$
							tuple.add(null);
							continue;
						}
						val = vals.get(index);
						try {
							tuple.add(DataTypeManager.transformValue(val, table.getColumns().get(output).getSymbol().getType()));
						} catch (TransformationException e) {
							throw new TeiidProcessingException(QueryPlugin.Event.TEIID30176, e, QueryPlugin.Util.gs(QueryPlugin.Event.TEIID30176, col.getName(), textLine, systemId));
						}
					}
					addBatchRow(tuple);
					if (rowNumber == limit) {
						terminateBatches();
						break;
					}
				} catch (Exception e) {
					errorCount++;
					try {
						Boolean handled = (Boolean) DVCaller.callWithExc(138, getContext(), table, errorCount, textLine, line.toString(), e);
						if (handled == null || !handled)
							throw e;
					} catch (Throwable callerExc) {
						if (callerExc instanceof TeiidProcessingException)
							throw (TeiidProcessingException) callerExc;
						else
							throw new TeiidProcessingException(callerExc);
					}
				}
			}
		}
	}

	// EXTENSION
	// ISSUE: DV-125
	// FIXED BY: Roman Pedchenko
	// DATE: 2014.02.24
	private char[] sb = null;
	// END EXTENSION

	private StringBuilder readLine(int maxLength, boolean exact, boolean invalue) throws TeiidProcessingException {
		if (eof) {
			return null;
		}
		// EXTENSION
		// ISSUE: DVCORE-444,DVCORE-589,DV-125
		// FIXED BY: Roman Pedchenko
		// DATE: 2013.05.23,2013.08.06,2014.02.24
//		StringBuilder sb = new StringBuilder(exact ? maxLength : (maxLength >> 4));
		if (sb == null || sb.length < maxLength + 1)
			sb = new char[maxLength + 1];
		int sbLength = 0;
		if (invalue) {
			//we must include the newline in the quoted value
			//sb.insert(0, newLine);
			sb[0] = newLine;
			sbLength++;
		}
		while (true) {
			char c = readChar();
			if (c == newLine) {
				if (this.noQuote && table.isUsingRowDelimiter()) {
					if (!eof && sbLength > 0 && sb[sbLength - 1] == this.quote) {
						// previous char was the escape -> current newLine is a literal char inside the row
						sb[sbLength++] = c;
						continue; // do not terminate the row here
					}
				}
//				if (sb.length() == 0) {
				if (sbLength == 0) {
					if (eof) {
						return null;
					}
					if (table.isUsingRowDelimiter()) {
						continue; //skip empty lines
					}
				}
				if (table.isUsingRowDelimiter()) {
//				    return sb.toString();
					return new StringBuilder(new String(sb, 0, sbLength));
				}
		    }
//			sb.append(c);
		    sb[sbLength++] = c;
//		    if (exact && sb.length() == maxLength && !table.isUsingRowDelimiter()) {
//		    	return sb.toString();
//		    }
		    if (exact && sbLength == maxLength && !table.isUsingRowDelimiter()) {
		    	return new StringBuilder(new String(sb, 0, sbLength));
		    }
//		    if (sb.length() > maxLength) {
		    if (sbLength > maxLength) {
		    	if (exact) {
//		    		sb.deleteCharAt(sb.length() - 1);
		    		sbLength--;
		    		//we're not forcing them to fully specify the line, so just drop the rest
		    		//TODO: there should be a max read length
				    while (readChar() != newLine) {

		    		}
//		    		return sb.toString();
		    		return new StringBuilder(new String(sb, 0, sbLength));
		    	}
			    //protects non-fixed width processing from run-away values
			    //TODO it is possible that string values could be desired that are longer than the max and/or returned as clobs
		    	throw new TeiidProcessingException(QueryPlugin.Event.TEIID30178, QueryPlugin.Util.gs(QueryPlugin.Event.TEIID30178, textLine+1, systemId, maxLength));
		    }
		}
		// END EXTENSION
	}

	private char readChar() throws TeiidProcessingException {
		try {
			int c = reader.read();
		    if (cr) {
			    if (c == newLine) {
				    c = reader.read();
				}
				cr = false;
		    }
		    switch (c) {
			    case '\r':
				    if (crNewLine) {
					    cr = true;
					    textLine++;
					    return newLine;
				    }
				    break;
			    case -1:
				    eof = true;
				    textLine++;
				    return newLine;
		    }
			if (c == newLine) {
				textLine++;
				return newLine;
		    }
		    return (char)c;
		// EXTENSION
		// ISSUE: DVCORE-2708
		// FIXED BY: Roman Pedchenko
		// DATE: 2015.03.12
		} catch (MalformedInputException e) {
			 throw new TeiidProcessingException(QueryPlugin.Event.TEIID30179, e, QueryPlugin.Util.getString(QueryPlugin.Event.DVCORE301258.toString(), e.toString(), textLine));
		// END EXTENSION
		} catch (IOException e) {
			 throw new TeiidProcessingException(QueryPlugin.Event.TEIID30179, e);
		}
	}

	private void initReader() throws ExpressionEvaluationException,
			BlockedException, TeiidComponentException, TeiidProcessingException {

		setReferenceValues(this.table);
		ClobType file = (ClobType)getEvaluator(Collections.emptyMap()).evaluate(table.getFile(), null);
		if (file == null) {
			return;
		}

		//get the reader
		try {
			this.systemId = "Unknown"; //$NON-NLS-1$
			if (file.getReference() instanceof ClobImpl) {
				this.systemId = ((ClobImpl)file.getReference()).getStreamFactory().getSystemId();
				if (this.systemId == null) {
					this.systemId = "Unknown"; //$NON-NLS-1$
				}
			}
			Reader r = file.getCharacterStream();
			if (!(r instanceof BufferedReader)) {
				reader = new BufferedReader(r);
			} else {
				reader = (BufferedReader)r;
			}
		} catch (SQLException e) {
			 throw new TeiidProcessingException(QueryPlugin.Event.TEIID30180, e);
		}

		//process the skip field
		if (skip <= 0) {
			return;
		}
		while (textLine < skip) {
			boolean isHeader = textLine == header;
			if (isHeader) {
				// EXTENSION
				// ISSUE: DVCORE-2607
				// FIXED BY: Roman Pedchenko
				// DATE: 2015.02.17
				// String line = readLine(DataTypeManager.MAX_STRING_LENGTH * 16, false);
				StringBuilder line = readLine(maxWidth != null ? maxWidth : DataTypeManager.getEstimatedStringLength() * 16, false, false);
				// END EXTENSION
				if (line == null) { //just return an empty batch
					reset();
					return;
				}
				processHeader(parseLine(line));
			} else {
				while (readChar() != newLine) {

	    		}
			}
		}
	}

	private void processHeader(List<String> line) {
		nameIndexes = new HashMap<String, Integer>();
		// EXTENSION
		// ISSUE: DVCORE-2607
		// FIXED BY: Roman Pedchenko
		// DATE: 2015.02.17
		// this.lineWidth = DataTypeManager.MAX_STRING_LENGTH * line.size();
		this.lineWidth = maxWidth != null ? maxWidth : DataTypeManager.getEstimatedStringLength() * line.size();
		// END EXTENSION
		for (String string : line) {
			if (string == null) {
				continue;
			}
			nameIndexes.put(string.toUpperCase(), nameIndexes.size());
		}
		for (TextColumn col : table.getColumns()) {
			if (col.isOrdinal()) {
				continue;
			}
			String name = col.getName().toUpperCase();
			if (col.getHeader() != null) {
				name = col.getHeader().toUpperCase();
			}
			Integer index = nameIndexes.get(name);
			if (index == null) {
				getContext().addWarning(new TeiidProcessingException(QueryPlugin.Event.TEIID30181, QueryPlugin.Util.gs(QueryPlugin.Event.TEIID30181, col.getName(), systemId)));
			}
			nameIndexes.put(col.getName(), index);
		}
	}

	private List<String> parseLine(StringBuilder line) throws TeiidProcessingException {
		if (table.isFixedWidth()) {
			return parseFixedWidth(line);
		}
		return parseDelimitedLine(line);
	}

	// EXTENSION
	// ISSUE: DV-125
	// FIXED BY: Roman Pedchenko
	// DATE: 2014.02.24
	private char[] cbuf = null;
	// END EXTENSION

	private List<String> parseDelimitedLine(StringBuilder line) throws TeiidProcessingException {
		ArrayList<String> result = new ArrayList<String>();
		StringBuilder builder = new StringBuilder();
		// EXTENSION
		// ISSUE: DVCORE-444,DV-125
		// FIXED BY: Roman Pedchenko
		// DATE: 2013.05.24
		int cbufLength = line.length();
		if (cbuf == null || cbuf.length < cbufLength)
			cbuf = new char[cbufLength];
		int charCount = 0;
		boolean escaped = false;
		boolean wasQualified = false;
		boolean qualified = false;
		while (true) {
			if (line == null) {
				if (escaped) {
					/* allow for escaped row delimiter/new line; do not inject newLine at EOF */
					escaped = false;
					StringBuilder next = readLine(lineWidth, false, false);
					int nextLen = next == null ? 0 : next.length();
					if (next == null) {
						// EOF: close current entry without injecting a trailing delimiter
						builder.append(cbuf, 0, charCount);
						addValue(result, wasQualified || noTrim, builder.toString());
						return result;
					}
					if (cr) {
//						builder.append('\r');
						cbuf[charCount++] = '\r'; if (charCount >= cbufLength) { builder.append(cbuf, 0, charCount); charCount = 0; }
					}
//					builder.append(newLine);
					cbuf[charCount++] = newLine; if (charCount >= cbufLength) { builder.append(cbuf, 0, charCount); charCount = 0; }
					line = next;
					cbufLength = nextLen;
					if (cbuf == null || cbuf.length < cbufLength)
						cbuf = new char[cbufLength];
					continue;
				}
				if (!qualified) {
					//close the last entry
					builder.append(cbuf, 0, charCount);
					addValue(result, wasQualified || noTrim, builder.toString());
					return result;
				}
				line = readLine(lineWidth, false, true);
				if (line == null) {
					 throw new TeiidProcessingException(QueryPlugin.Event.TEIID30182, QueryPlugin.Util.gs(QueryPlugin.Event.TEIID30182, systemId));
				}
				cbufLength = line.length();
				if (cbuf == null || cbuf.length < cbufLength)
					cbuf = new char[cbufLength];
			}
			for (int i = 0; i < line.length(); i++) {
				char chr = line.charAt(i);
				if (chr == delimiter) {
					if (escaped || qualified) {
//						builder.append(chr);
						cbuf[charCount++] = chr; if (charCount >= cbufLength) { builder.append(cbuf, 0, charCount); charCount = 0; }
						escaped = false;
					} else {
						builder.append(cbuf, 0, charCount);
						charCount = 0;
						addValue(result, wasQualified || noTrim, builder.toString());
						wasQualified = false;
//						builder = new StringBuilder();  //next entry
						builder.setLength(0);
					}
				// EXTENSION
				// ISSUE: DVCORE-1167
				// FIXED BY: Roman Pedchenko
				// DATE: 2014.07.10
				// } else if (chr == quote) {
				} else if (quote != '\0' && chr == quote) {
				// END EXTENSION
					if (noQuote) { 	//it's the escape char
						if (!(escaped = (i + 1 >= cbufLength))) {
							char nextChar = line.charAt(i + 1);
							if (!(escaped = (nextChar == delimiter || nextChar == newLine))) {
								cbuf[charCount++] = quote; if (charCount >= cbufLength) {builder.append(cbuf, 0, charCount); charCount = 0;}

								// double escape: skip one char
								if (nextChar == quote) i++;
							}
						}
					} else {
						if (qualified) {
							qualified = false;
						} else {
							if (wasQualified) {
								qualified = true;
//								builder.append(chr);
								cbuf[charCount++] = chr; if (charCount >= cbufLength) { builder.append(cbuf, 0, charCount); charCount = 0; }
							} else {
								builder.append(cbuf, 0, charCount);
								charCount = 0;
								if (builder.toString().trim().length() != 0) {
									 throw new TeiidProcessingException(QueryPlugin.Event.TEIID30183, QueryPlugin.Util.gs(QueryPlugin.Event.TEIID30183, textLine, systemId));
								}
								qualified = true;
//								builder = new StringBuilder(); //start the entry over
								builder.setLength(0);
								wasQualified = true;
							}
						}
					}
				} else {
					if (escaped) {
						//don't understand other escape sequences yet
//						 throw new TeiidProcessingException(QueryPlugin.Event.TEIID30184, QueryPlugin.Util.gs(QueryPlugin.Event.TEIID30184, chr, textLine, systemId));
					}
					if (wasQualified && !qualified) {
						if (!Character.isWhitespace(chr)) {
							 throw new TeiidProcessingException(QueryPlugin.Event.TEIID30183, QueryPlugin.Util.gs(QueryPlugin.Event.TEIID30183, textLine, systemId));
						}
						//else just ignore
					} else {
//						builder.append(chr);
						cbuf[charCount++] = chr; if (charCount >= cbufLength) { builder.append(cbuf, 0, charCount); charCount = 0; }
					}
				}
			}
			line = null;
		}
		// END EXTENSION
	}

	private void addValue(ArrayList<String> result, boolean wasQualified, String val) {
		if (!wasQualified) {
			val = val.trim();
			if (val.length() == 0) {
				val = null;
			}
		}
		result.add(val);
	}

	private List<String> parseFixedWidth(StringBuilder line) {
		ArrayList<String> result = new ArrayList<String>();
		int beginIndex = 0;
		for (TextColumn col : table.getColumns()) {
			if (beginIndex >= line.length() || col.getWidth() == null) {
				result.add(null);
			} else {
				String val = new String(line.substring(beginIndex, Math.min(line.length(), beginIndex + col.getWidth())));
				addValue(result, col.isNoTrim(), val);
				beginIndex += col.getWidth();
			}
		}
		return result;
	}

	@Override
	public Collection<? extends LanguageObject> getObjects() {
		return Arrays.asList(this.table.getFile());
	}

}
