    001/*
    002 * Copyright (c) 2016-2017 Chris K Wensel <chris@wensel.net>. All Rights Reserved.
    003 * Copyright (c) 2007-2017 Xplenty, Inc. All Rights Reserved.
    004 *
    005 * Project and contact information: http://www.cascading.org/
    006 *
    007 * This file is part of the Cascading project.
    008 *
    009 * Licensed under the Apache License, Version 2.0 (the "License");
    010 * you may not use this file except in compliance with the License.
    011 * You may obtain a copy of the License at
    012 *
    013 *     http://www.apache.org/licenses/LICENSE-2.0
    014 *
    015 * Unless required by applicable law or agreed to in writing, software
    016 * distributed under the License is distributed on an "AS IS" BASIS,
    017 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    018 * See the License for the specific language governing permissions and
    019 * limitations under the License.
    020 */
    021
    022package cascading.scheme.local;
    023
    024import java.beans.ConstructorProperties;
    025import java.io.IOException;
    026import java.io.InputStream;
    027import java.io.InputStreamReader;
    028import java.io.LineNumberReader;
    029import java.io.OutputStream;
    030import java.io.OutputStreamWriter;
    031import java.io.PrintWriter;
    032import java.io.UnsupportedEncodingException;
    033import java.nio.charset.Charset;
    034import java.util.Properties;
    035
    036import cascading.flow.FlowProcess;
    037import cascading.management.annotation.Property;
    038import cascading.management.annotation.PropertyDescription;
    039import cascading.management.annotation.Visibility;
    040import cascading.scheme.FileFormat;
    041import cascading.scheme.Scheme;
    042import cascading.scheme.SinkCall;
    043import cascading.scheme.SourceCall;
    044import cascading.scheme.util.DelimitedParser;
    045import cascading.tap.CompositeTap;
    046import cascading.tap.Tap;
    047import cascading.tap.TapException;
    048import cascading.tap.local.FileTap;
    049import cascading.tuple.Fields;
    050import cascading.tuple.Tuple;
    051import cascading.tuple.TupleEntry;
    052import cascading.tuple.util.TupleViews;
    053
    054/**
    055 * Class TextDelimited provides direct support for delimited text files, like
    056 * TAB (\t) or COMMA (,) delimited files. It also optionally allows for quoted values.
    057 * <p/>
    058 * TextDelimited may also be used to skip the "header" in a file, where the header is defined as the very first line
    059 * in every input file. That is, if the byte offset of the current line from the input is zero (0), that line will
    060 * be skipped.
    061 * <p/>
    062 * It is assumed if sink/source {@code fields} is set to either {@link Fields#ALL} or {@link Fields#UNKNOWN} and
    063 * {@code skipHeader} or {@code hasHeader} is {@code true}, the field names will be retrieved from the header of the
    064 * file and used during planning. The header will parsed with the same rules as the body of the file.
    065 * <p/>
    066 * By default headers are not skipped.
    067 * <p/>
    068 * TextDelimited may also be used to write a "header" in a file. The fields names for the header are taken directly
    069 * from the declared fields. Or if the declared fields are {@link Fields#ALL} or {@link Fields#UNKNOWN}, the
    070 * resolved field names will be used, if any.
    071 * <p/>
    072 * By default headers are not written.
    073 * <p/>
    074 * If {@code hasHeaders} is set to {@code true} on a constructor, both {@code skipHeader} and {@code writeHeader} will
    075 * be set to {@code true}.
    076 * <p/>
    077 * By default this {@link cascading.scheme.Scheme} is both {@code strict} and {@code safe}.
    078 * <p/>
    079 * Strict meaning if a line of text does not parse into the expected number of fields, this class will throw a
    080 * {@link TapException}. If strict is {@code false}, then {@link Tuple} will be returned with {@code null} values
    081 * for the missing fields.
    082 * <p/>
    083 * Safe meaning if a field cannot be coerced into an expected type, a {@code null} will be used for the value.
    084 * If safe is {@code false}, a {@link TapException} will be thrown.
    085 * <p/>
    086 * Also by default, {@code quote} strings are not searched for to improve processing speed. If a file is
    087 * COMMA delimited but may have COMMA's in a value, the whole value should be surrounded by the quote string, typically
    088 * double quotes ({@literal "}).
    089 * <p/>
    090 * Note all empty fields in a line will be returned as {@code null} unless coerced into a new type.
    091 * <p/>
    092 * This Scheme may source/sink {@link Fields#ALL}, when given on the constructor the new instance will automatically
    093 * default to strict == false as the number of fields parsed are arbitrary or unknown. A type array may not be given
    094 * either, so all values will be returned as Strings.
    095 * <p/>
    096 * By default, all text is encoded/decoded as UTF-8. This can be changed via the {@code charsetName} constructor
    097 * argument.
    098 * <p/>
    099 * To override field and line parsing behaviors, sub-class {@link DelimitedParser} or provide a
    100 * {@link cascading.scheme.util.FieldTypeResolver} implementation.
    101 * <p/>
    102 * Note that there should be no expectation that TextDelimited, or specifically {@link DelimitedParser}, can handle
    103 * all delimited and quoted combinations reliably. Attempting to do so would impair its performance and maintainability.
    104 * <p/>
    105 * Further, it can be safely said any corrupted files will not be supported for obvious reasons. Corrupted files may
    106 * result in exceptions or could cause edge cases in the underlying java regular expression engine.
    107 * <p/>
    108 * A large part of Cascading was designed to help users cleans data. Thus the recommendation is to create Flows that
    109 * are responsible for cleansing large data-sets when faced with the problem
    110 * <p/>
    111 * DelimitedParser maybe sub-classed and extended if necessary.
    112 *
    113 * @see TextLine
    114 */
    115public class TextDelimited extends Scheme<Properties, InputStream, OutputStream, LineNumberReader, PrintWriter> implements FileFormat
    116  {
    117  public static final String DEFAULT_CHARSET = "UTF-8";
    118
    119  private final boolean skipHeader;
    120  private final boolean writeHeader;
    121  private final DelimitedParser delimitedParser;
    122  private String charsetName = DEFAULT_CHARSET;
    123
    124  /**
    125   * Constructor TextDelimited creates a new TextDelimited instance sourcing {@link Fields#UNKNOWN}, sinking
    126   * {@link Fields#ALL} and using TAB as the default delimiter.
    127   * <p/>
    128   * Use this constructor if the source and sink fields will be resolved during planning, for example, when using
    129   * with a {@link cascading.pipe.Checkpoint} Tap.
    130   */
    131  public TextDelimited()
    132    {
    133    this( Fields.ALL );
    134    }
    135
    136  /**
    137   * Constructor TextDelimited creates a new TextDelimited instance sourcing {@link Fields#UNKNOWN}, sinking
    138   * {@link Fields#ALL} and using TAB as the default delimiter.
    139   * <p/>
    140   * Use this constructor if the source and sink fields will be resolved during planning, for example, when using
    141   * with a {@link cascading.pipe.Checkpoint} Tap.
    142   *
    143   * @param hasHeader
    144   * @param delimiter
    145   */
    146  @ConstructorProperties({"hasHeader", "delimiter"})
    147  public TextDelimited( boolean hasHeader, String delimiter )
    148    {
    149    this( Fields.ALL, hasHeader, delimiter, null, (Class[]) null );
    150    }
    151
    152  /**
    153   * Constructor TextDelimited creates a new TextDelimited instance sourcing {@link Fields#UNKNOWN}, sinking
    154   * {@link Fields#ALL} and using TAB as the default delimiter.
    155   * <p/>
    156   * Use this constructor if the source and sink fields will be resolved during planning, for example, when using
    157   * with a {@link cascading.pipe.Checkpoint} Tap.
    158   *
    159   * @param hasHeader
    160   * @param delimiter
    161   * @param quote
    162   */
    163  @ConstructorProperties({"hasHeader", "delimiter", "quote"})
    164  public TextDelimited( boolean hasHeader, String delimiter, String quote )
    165    {
    166    this( Fields.ALL, hasHeader, delimiter, quote, (Class[]) null );
    167    }
    168
    169  /**
    170   * Constructor TextDelimited creates a new TextDelimited instance sourcing {@link Fields#UNKNOWN}, sinking
    171   * {@link Fields#ALL} and using the given delimitedParser instance for parsing.
    172   * <p/>
    173   * Use this constructor if the source and sink fields will be resolved during planning, for example, when using
    174   * with a {@link cascading.pipe.Checkpoint} Tap.
    175   *
    176   * @param hasHeader
    177   * @param delimitedParser
    178   */
    179  @ConstructorProperties({"hasHeader", "delimitedParser"})
    180  public TextDelimited( boolean hasHeader, DelimitedParser delimitedParser )
    181    {
    182    this( Fields.ALL, hasHeader, hasHeader, delimitedParser );
    183    }
    184
    185  /**
    186   * Constructor TextDelimited creates a new TextDelimited instance sourcing {@link Fields#UNKNOWN}, sinking
    187   * {@link Fields#ALL} and using the given delimitedParser instance for parsing.
    188   * <p/>
    189   * Use this constructor if the source and sink fields will be resolved during planning, for example, when using
    190   * with a {@link cascading.pipe.Checkpoint} Tap.
    191   * <p/>
    192   * This constructor will set {@code skipHeader} and {@code writeHeader} values to true.
    193   *
    194   * @param delimitedParser
    195   */
    196  @ConstructorProperties({"delimitedParser"})
    197  public TextDelimited( DelimitedParser delimitedParser )
    198    {
    199    this( Fields.ALL, true, true, delimitedParser );
    200    }
    201
    202  /**
    203   * Constructor TextDelimited creates a new TextDelimited instance with TAB as the default delimiter.
    204   *
    205   * @param fields of type Fields
    206   */
    207  @ConstructorProperties({"fields"})
    208  public TextDelimited( Fields fields )
    209    {
    210    this( fields, "\t", null, null );
    211    }
    212
    213  /**
    214   * Constructor TextDelimited creates a new TextDelimited instance.
    215   *
    216   * @param fields    of type Fields
    217   * @param delimiter of type String
    218   */
    219  @ConstructorProperties({"fields", "delimiter"})
    220  public TextDelimited( Fields fields, String delimiter )
    221    {
    222    this( fields, delimiter, null, null );
    223    }
    224
    225  /**
    226   * Constructor TextDelimited creates a new TextDelimited instance.
    227   *
    228   * @param fields    of type Fields
    229   * @param hasHeader of type boolean
    230   * @param delimiter of type String
    231   */
    232  @ConstructorProperties({"fields", "hasHeader", "delimiter"})
    233  public TextDelimited( Fields fields, boolean hasHeader, String delimiter )
    234    {
    235    this( fields, hasHeader, hasHeader, delimiter, null, null );
    236    }
    237
    238  /**
    239   * Constructor TextDelimited creates a new TextDelimited instance.
    240   *
    241   * @param fields     of type Fields
    242   * @param skipHeader of type boolean
    243   * @param delimiter  of type String
    244   */
    245  @ConstructorProperties({"fields", "skipHeader", "writeHeader", "delimiter"})
    246  public TextDelimited( Fields fields, boolean skipHeader, boolean writeHeader, String delimiter )
    247    {
    248    this( fields, skipHeader, writeHeader, delimiter, null, null );
    249    }
    250
    251  /**
    252   * Constructor TextDelimited creates a new TextDelimited instance.
    253   *
    254   * @param fields    of type Fields
    255   * @param delimiter of type String
    256   * @param types     of type Class[]
    257   */
    258  @ConstructorProperties({"fields", "delimiter", "types"})
    259  public TextDelimited( Fields fields, String delimiter, Class[] types )
    260    {
    261    this( fields, delimiter, null, types );
    262    }
    263
    264  /**
    265   * Constructor TextDelimited creates a new TextDelimited instance.
    266   *
    267   * @param fields    of type Fields
    268   * @param hasHeader of type boolean
    269   * @param delimiter of type String
    270   * @param types     of type Class[]
    271   */
    272  @ConstructorProperties({"fields", "hasHeader", "delimiter", "types"})
    273  public TextDelimited( Fields fields, boolean hasHeader, String delimiter, Class[] types )
    274    {
    275    this( fields, hasHeader, hasHeader, delimiter, null, types );
    276    }
    277
    278  /**
    279   * Constructor TextDelimited creates a new TextDelimited instance.
    280   *
    281   * @param fields      of type Fields
    282   * @param skipHeader  of type boolean
    283   * @param writeHeader of type boolean
    284   * @param delimiter   of type String
    285   * @param types       of type Class[]
    286   */
    287  @ConstructorProperties({"fields", "skipHeader", "writeHeader", "delimiter", "types"})
    288  public TextDelimited( Fields fields, boolean skipHeader, boolean writeHeader, String delimiter, Class[] types )
    289    {
    290    this( fields, skipHeader, writeHeader, delimiter, null, types );
    291    }
    292
    293  /**
    294   * Constructor TextDelimited creates a new TextDelimited instance.
    295   *
    296   * @param fields    of type Fields
    297   * @param delimiter of type String
    298   * @param quote     of type String
    299   * @param types     of type Class[]
    300   */
    301  @ConstructorProperties({"fields", "delimiter", "quote", "types"})
    302  public TextDelimited( Fields fields, String delimiter, String quote, Class[] types )
    303    {
    304    this( fields, false, delimiter, quote, types );
    305    }
    306
    307  /**
    308   * Constructor TextDelimited creates a new TextDelimited instance.
    309   *
    310   * @param fields    of type Fields
    311   * @param hasHeader of type boolean
    312   * @param delimiter of type String
    313   * @param quote     of type String
    314   * @param types     of type Class[]
    315   */
    316  @ConstructorProperties({"fields", "hasHeader", "delimiter", "quote", "types"})
    317  public TextDelimited( Fields fields, boolean hasHeader, String delimiter, String quote, Class[] types )
    318    {
    319    this( fields, hasHeader, hasHeader, delimiter, quote, types, true );
    320    }
    321
    322  /**
    323   * Constructor TextDelimited creates a new TextDelimited instance.
    324   *
    325   * @param fields      of type Fields
    326   * @param skipHeader  of type boolean
    327   * @param writeHeader of type boolean
    328   * @param delimiter   of type String
    329   * @param quote       of type String
    330   * @param types       of type Class[]
    331   */
    332  @ConstructorProperties({"fields", "skipHeader", "writeHeader", "delimiter", "quote", "types"})
    333  public TextDelimited( Fields fields, boolean skipHeader, boolean writeHeader, String delimiter, String quote, Class[] types )
    334    {
    335    this( fields, skipHeader, writeHeader, delimiter, quote, types, true );
    336    }
    337
    338  /**
    339   * Constructor TextDelimited creates a new TextDelimited instance.
    340   *
    341   * @param fields    of type Fields
    342   * @param delimiter of type String
    343   * @param quote     of type String
    344   * @param types     of type Class[]
    345   * @param safe      of type boolean
    346   */
    347  @ConstructorProperties({"fields", "delimiter", "quote", "types", "safe"})
    348  public TextDelimited( Fields fields, String delimiter, String quote, Class[] types, boolean safe )
    349    {
    350    this( fields, false, delimiter, quote, types, safe );
    351    }
    352
    353  /**
    354   * Constructor TextDelimited creates a new TextDelimited instance.
    355   *
    356   * @param fields    of type Fields
    357   * @param hasHeader of type boolean
    358   * @param delimiter of type String
    359   * @param quote     of type String
    360   * @param types     of type Class[]
    361   * @param safe      of type boolean
    362   */
    363  @ConstructorProperties({"fields", "hasHeader", "delimiter", "quote", "types", "safe"})
    364  public TextDelimited( Fields fields, boolean hasHeader, String delimiter, String quote, Class[] types, boolean safe )
    365    {
    366    this( fields, hasHeader, hasHeader, delimiter, true, quote, types, safe );
    367    }
    368
    369  /**
    370   * Constructor TextDelimited creates a new TextDelimited instance.
    371   *
    372   * @param fields      of type Fields
    373   * @param hasHeader   of type boolean
    374   * @param delimiter   of type String
    375   * @param quote       of type String
    376   * @param types       of type Class[]
    377   * @param safe        of type boolean
    378   * @param charsetName of type String
    379   */
    380  @ConstructorProperties({"fields", "hasHeader", "delimiter", "quote", "types", "safe", "charsetName"})
    381  public TextDelimited( Fields fields, boolean hasHeader, String delimiter, String quote, Class[] types, boolean safe, String charsetName )
    382    {
    383    this( fields, hasHeader, hasHeader, delimiter, true, quote, types, safe, charsetName );
    384    }
    385
    386  /**
    387   * Constructor TextDelimited creates a new TextDelimited instance.
    388   *
    389   * @param fields      of type Fields
    390   * @param skipHeader  of type boolean
    391   * @param writeHeader of type boolean
    392   * @param delimiter   of type String
    393   * @param quote       of type String
    394   * @param types       of type Class[]
    395   * @param safe        of type boolean
    396   */
    397  @ConstructorProperties({"fields", "skipHeader", "writeHeader", "delimiter", "quote", "types", "safe"})
    398  public TextDelimited( Fields fields, boolean skipHeader, boolean writeHeader, String delimiter, String quote, Class[] types, boolean safe )
    399    {
    400    this( fields, skipHeader, writeHeader, delimiter, true, quote, types, safe );
    401    }
    402
    403  /**
    404   * Constructor TextDelimited creates a new TextDelimited instance.
    405   *
    406   * @param fields    of type Fields
    407   * @param delimiter of type String
    408   * @param quote     of type String
    409   */
    410  @ConstructorProperties({"fields", "delimiter", "quote"})
    411  public TextDelimited( Fields fields, String delimiter, String quote )
    412    {
    413    this( fields, false, delimiter, quote, null, true );
    414    }
    415
    416  /**
    417   * Constructor TextDelimited creates a new TextDelimited instance.
    418   *
    419   * @param fields    of type Fields
    420   * @param hasHeader of type boolean
    421   * @param delimiter of type String
    422   * @param quote     of type String
    423   */
    424  @ConstructorProperties({"fields", "hasHeader", "delimiter", "quote"})
    425  public TextDelimited( Fields fields, boolean hasHeader, String delimiter, String quote )
    426    {
    427    this( fields, hasHeader, delimiter, quote, null, true );
    428    }
    429
    430  /**
    431   * Constructor TextDelimited creates a new TextDelimited instance.
    432   *
    433   * @param fields      of type Fields
    434   * @param hasHeader   of type boolean
    435   * @param delimiter   of type String
    436   * @param quote       of type String
    437   * @param charsetName of type String
    438   */
    439  @ConstructorProperties({"fields", "hasHeader", "delimiter", "quote", "charsetName"})
    440  public TextDelimited( Fields fields, boolean hasHeader, String delimiter, String quote, String charsetName )
    441    {
    442    this( fields, hasHeader, delimiter, quote, null, true, charsetName );
    443    }
    444
    445  /**
    446   * Constructor TextDelimited creates a new TextDelimited instance.
    447   *
    448   * @param fields      of type Fields
    449   * @param skipHeader  of type boolean
    450   * @param writeHeader of type boolean
    451   * @param delimiter   of type String
    452   * @param strict      of type boolean
    453   * @param quote       of type String
    454   * @param types       of type Class[]
    455   * @param safe        of type boolean
    456   */
    457  @ConstructorProperties({"fields", "skipHeader", "writeHeader", "delimiter", "strict", "quote", "types", "safe"})
    458  public TextDelimited( Fields fields, boolean skipHeader, boolean writeHeader, String delimiter, boolean strict, String quote, Class[] types, boolean safe )
    459    {
    460    this( fields, skipHeader, writeHeader, delimiter, strict, quote, types, safe, DEFAULT_CHARSET );
    461    }
    462
    463  /**
    464   * Constructor TextDelimited creates a new TextDelimited instance.
    465   *
    466   * @param fields      of type Fields
    467   * @param skipHeader  of type boolean
    468   * @param writeHeader of type boolean
    469   * @param delimiter   of type String
    470   * @param strict      of type boolean
    471   * @param quote       of type String
    472   * @param types       of type Class[]
    473   * @param safe        of type boolean
    474   * @param charsetName of type String
    475   */
    476  @ConstructorProperties({"fields", "skipHeader", "writeHeader", "delimiter", "strict", "quote", "types", "safe",
    477                          "charsetName"})
    478  public TextDelimited( Fields fields, boolean skipHeader, boolean writeHeader, String delimiter, boolean strict, String quote, Class[] types, boolean safe, String charsetName )
    479    {
    480    this( fields, skipHeader, writeHeader, charsetName, new DelimitedParser( delimiter, quote, types, strict, safe ) );
    481    }
    482
    483  /**
    484   * Constructor TextDelimited creates a new TextDelimited instance.
    485   *
    486   * @param fields          of type Fields
    487   * @param writeHeader     of type boolean
    488   * @param delimitedParser of type DelimitedParser
    489   */
    490  @ConstructorProperties({"fields", "skipHeader", "writeHeader", "delimitedParser"})
    491  public TextDelimited( Fields fields, boolean skipHeader, boolean writeHeader, DelimitedParser delimitedParser )
    492    {
    493    this( fields, skipHeader, writeHeader, null, delimitedParser );
    494    }
    495
    496  /**
    497   * Constructor TextDelimited creates a new TextDelimited instance.
    498   *
    499   * @param fields          of type Fields
    500   * @param hasHeader       of type boolean
    501   * @param delimitedParser of type DelimitedParser
    502   */
    503  @ConstructorProperties({"fields", "hasHeader", "delimitedParser"})
    504  public TextDelimited( Fields fields, boolean hasHeader, DelimitedParser delimitedParser )
    505    {
    506    this( fields, hasHeader, hasHeader, null, delimitedParser );
    507    }
    508
    509  /**
    510   * Constructor TextDelimited creates a new TextDelimited instance.
    511   *
    512   * @param fields          of type Fields
    513   * @param writeHeader     of type boolean
    514   * @param charsetName     of type String
    515   * @param delimitedParser of type DelimitedParser
    516   */
    517  @ConstructorProperties({"fields", "skipHeader", "writeHeader", "charsetName", "delimitedParser"})
    518  public TextDelimited( Fields fields, boolean skipHeader, boolean writeHeader, String charsetName, DelimitedParser delimitedParser )
    519    {
    520    super( fields, fields );
    521
    522    this.delimitedParser = delimitedParser;
    523
    524    // normalizes ALL and UNKNOWN
    525    // calls reset on delimitedParser
    526    setSourceFields( fields );
    527    setSinkFields( fields );
    528
    529    this.skipHeader = skipHeader;
    530    this.writeHeader = writeHeader;
    531
    532    if( charsetName != null )
    533      this.charsetName = charsetName;
    534
    535    // throws an exception if not found
    536    Charset.forName( this.charsetName );
    537    }
    538
    539  @Property(name = "charset", visibility = Visibility.PUBLIC)
    540  @PropertyDescription("character set used.")
    541  public String getCharsetName()
    542    {
    543    return charsetName;
    544    }
    545
    546  /**
    547   * Method getDelimiter returns the delimiter used to parse fields from the current line of text.
    548   *
    549   * @return a String
    550   */
    551  @Property(name = "delimiter", visibility = Visibility.PUBLIC)
    552  @PropertyDescription("The delimiter used to separate fields.")
    553  public String getDelimiter()
    554    {
    555    return delimitedParser.getDelimiter();
    556    }
    557
    558  /**
    559   * Method getQuote returns the quote string, if any, used to encapsulate each field in a line to delimited text.
    560   *
    561   * @return a String
    562   */
    563  @Property(name = "quote", visibility = Visibility.PUBLIC)
    564  @PropertyDescription("The string used for quoting.")
    565  public String getQuote()
    566    {
    567    return delimitedParser.getQuote();
    568    }
    569
    570  public LineNumberReader createInput( InputStream inputStream )
    571    {
    572    try
    573      {
    574      return new LineNumberReader( new InputStreamReader( inputStream, charsetName ) );
    575      }
    576    catch( UnsupportedEncodingException exception )
    577      {
    578      throw new TapException( exception );
    579      }
    580    }
    581
    582  public PrintWriter createOutput( OutputStream outputStream )
    583    {
    584    try
    585      {
    586      return new PrintWriter( new OutputStreamWriter( outputStream, charsetName ) );
    587      }
    588    catch( UnsupportedEncodingException exception )
    589      {
    590      throw new TapException( exception );
    591      }
    592    }
    593
    594  @Override
    595  public void setSinkFields( Fields sinkFields )
    596    {
    597    super.setSourceFields( sinkFields );
    598    super.setSinkFields( sinkFields );
    599
    600    if( delimitedParser != null )
    601      delimitedParser.reset( getSourceFields(), getSinkFields() );
    602    }
    603
    604  @Override
    605  public void setSourceFields( Fields sourceFields )
    606    {
    607    super.setSourceFields( sourceFields );
    608    super.setSinkFields( sourceFields );
    609
    610    if( delimitedParser != null )
    611      delimitedParser.reset( getSourceFields(), getSinkFields() );
    612    }
    613
    614  @Override
    615  public boolean isSymmetrical()
    616    {
    617    return super.isSymmetrical() && skipHeader == writeHeader;
    618    }
    619
    620  @Override
    621  public Fields retrieveSourceFields( FlowProcess<? extends Properties> process, Tap tap )
    622    {
    623    if( !skipHeader || !getSourceFields().isUnknown() )
    624      return getSourceFields();
    625
    626    // no need to open them all
    627    if( tap instanceof CompositeTap )
    628      tap = (Tap) ( (CompositeTap) tap ).getChildTaps().next();
    629
    630    tap = new FileTap( new TextLine( new Fields( "line" ), charsetName ), tap.getIdentifier() );
    631
    632    setSourceFields( delimitedParser.parseFirstLine( process, tap ) );
    633
    634    return getSourceFields();
    635    }
    636
    637  @Override
    638  public void presentSourceFields( FlowProcess<? extends Properties> process, Tap tap, Fields fields )
    639    {
    640    // do nothing
    641    }
    642
    643  @Override
    644  public void presentSinkFields( FlowProcess<? extends Properties> flowProcess, Tap tap, Fields fields )
    645    {
    646    if( writeHeader )
    647      presentSinkFieldsInternal( fields );
    648    }
    649
    650  @Override
    651  public void sourceConfInit( FlowProcess<? extends Properties> flowProcess, Tap<Properties, InputStream, OutputStream> tap, Properties conf )
    652    {
    653    }
    654
    655  @Override
    656  public void sourcePrepare( FlowProcess<? extends Properties> flowProcess, SourceCall<LineNumberReader, InputStream> sourceCall ) throws IOException
    657    {
    658    sourceCall.setContext( createInput( sourceCall.getInput() ) );
    659
    660    sourceCall.getIncomingEntry().setTuple( TupleViews.createObjectArray() );
    661    }
    662
    663  @Override
    664  public void sourceRePrepare( FlowProcess<? extends Properties> flowProcess, SourceCall<LineNumberReader, InputStream> sourceCall ) throws IOException
    665    {
    666    sourceCall.setContext( createInput( sourceCall.getInput() ) );
    667    }
    668
    669  @Override
    670  public boolean source( FlowProcess<? extends Properties> flowProcess, SourceCall<LineNumberReader, InputStream> sourceCall ) throws IOException
    671    {
    672    String line = sourceCall.getContext().readLine();
    673
    674    if( line == null )
    675      return false;
    676
    677    if( skipHeader && sourceCall.getContext().getLineNumber() == 1 ) // todo: optimize this away
    678      line = sourceCall.getContext().readLine();
    679
    680    if( line == null )
    681      return false;
    682
    683    Object[] split = delimitedParser.parseLine( line );
    684
    685    // assumption it is better to re-use than to construct new
    686    Tuple tuple = sourceCall.getIncomingEntry().getTuple();
    687
    688    TupleViews.reset( tuple, split );
    689
    690    return true;
    691    }
    692
    693  @Override
    694  public void sourceCleanup( FlowProcess<? extends Properties> flowProcess, SourceCall<LineNumberReader, InputStream> sourceCall ) throws IOException
    695    {
    696    sourceCall.setContext( null );
    697    }
    698
    699  @Override
    700  public void sinkConfInit( FlowProcess<? extends Properties> flowProcess, Tap<Properties, InputStream, OutputStream> tap, Properties conf )
    701    {
    702    }
    703
    704  @Override
    705  public void sinkPrepare( FlowProcess<? extends Properties> flowProcess, SinkCall<PrintWriter, OutputStream> sinkCall )
    706    {
    707    sinkCall.setContext( createOutput( sinkCall.getOutput() ) );
    708
    709    if( writeHeader )
    710      {
    711      Fields fields = sinkCall.getOutgoingEntry().getFields();
    712      delimitedParser.joinFirstLine( fields, sinkCall.getContext() );
    713
    714      sinkCall.getContext().println();
    715      }
    716    }
    717
    718  @Override
    719  public void sink( FlowProcess<? extends Properties> flowProcess, SinkCall<PrintWriter, OutputStream> sinkCall ) throws IOException
    720    {
    721    TupleEntry tupleEntry = sinkCall.getOutgoingEntry();
    722
    723    Iterable<String> strings = tupleEntry.asIterableOf( String.class );
    724
    725    delimitedParser.joinLine( strings, sinkCall.getContext() );
    726
    727    sinkCall.getContext().println();
    728    }
    729
    730  @Override
    731  public void sinkCleanup( FlowProcess<? extends Properties> flowProcess, SinkCall<PrintWriter, OutputStream> sinkCall )
    732    {
    733    sinkCall.getContext().flush();
    734    sinkCall.setContext( null );
    735    }
    736
    737  @Override
    738  public String getExtension()
    739    {
    740    switch( getDelimiter().trim() )
    741      {
    742      case "\t":
    743        return "tsv";
    744
    745      case ",":
    746        return "csv";
    747      }
    748
    749    return "txt";
    750    }
    751  }