    001/*
    002 * Copyright (c) 2016-2017 Chris K Wensel <chris@wensel.net>. All Rights Reserved.
    003 * Copyright (c) 2007-2017 Xplenty, Inc. All Rights Reserved.
    004 *
    005 * Project and contact information: http://www.cascading.org/
    006 *
    007 * This file is part of the Cascading project.
    008 *
    009 * Licensed under the Apache License, Version 2.0 (the "License");
    010 * you may not use this file except in compliance with the License.
    011 * You may obtain a copy of the License at
    012 *
    013 *     http://www.apache.org/licenses/LICENSE-2.0
    014 *
    015 * Unless required by applicable law or agreed to in writing, software
    016 * distributed under the License is distributed on an "AS IS" BASIS,
    017 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    018 * See the License for the specific language governing permissions and
    019 * limitations under the License.
    020 */
    021
    022package cascading.scheme.hadoop;
    023
    024import java.beans.ConstructorProperties;
    025import java.io.IOException;
    026import java.nio.charset.Charset;
    027import java.util.Arrays;
    028
    029import cascading.flow.FlowProcess;
    030import cascading.flow.hadoop.util.HadoopUtil;
    031import cascading.management.annotation.Property;
    032import cascading.management.annotation.PropertyDescription;
    033import cascading.management.annotation.Visibility;
    034import cascading.scheme.FileFormat;
    035import cascading.scheme.Scheme;
    036import cascading.scheme.SinkCall;
    037import cascading.scheme.SourceCall;
    038import cascading.tap.Tap;
    039import cascading.tuple.Fields;
    040import cascading.tuple.Tuple;
    041import cascading.tuple.TupleEntry;
    042import org.apache.hadoop.conf.Configuration;
    043import org.apache.hadoop.fs.Path;
    044import org.apache.hadoop.io.LongWritable;
    045import org.apache.hadoop.io.Text;
    046import org.apache.hadoop.mapred.FileInputFormat;
    047import org.apache.hadoop.mapred.InputFormat;
    048import org.apache.hadoop.mapred.OutputCollector;
    049import org.apache.hadoop.mapred.OutputFormat;
    050import org.apache.hadoop.mapred.RecordReader;
    051import org.apache.hadoop.mapred.TextInputFormat;
    052import org.apache.hadoop.mapred.TextOutputFormat;
    053
    054import static cascading.flow.hadoop.util.HadoopUtil.asJobConfInstance;
    055
    056/**
    057 * A TextLine is a type of {@link cascading.scheme.Scheme} for plain text files. Files are broken into
    058 * lines. Either line-feed or carriage-return are used to signal end of line.
    059 * <p/>
    060 * By default, this scheme returns a {@link Tuple} with two fields, "offset" and "line".
    061 * <p/>
    062 * Many of the constructors take both "sourceFields" and "sinkFields". sourceFields denote the field names
    063 * to be used instead of the names "offset" and "line". sinkFields is a selector and is by default {@link Fields#ALL}.
    064 * Any available field names can be given if only a subset of the incoming fields should be used.
    065 * <p/>
    066 * If a {@link Fields} instance is passed on the constructor as sourceFields having only one field, the return tuples
    067 * will simply be the "line" value using the given field name.
    068 * <p/>
    069 * Note that TextLine will concatenate all the Tuple values for the selected fields with a TAB delimiter before
    070 * writing out the line.
    071 * <p/>
    072 * Note sink compression is {@link Compress#DISABLE} by default. If {@code null} is passed to the constructor
    073 * for the compression value, it will remain disabled.
    074 * <p/>
    075 * If any of the input files end with ".zip", an error will be thrown.
    076 * * <p/>
    077 * By default, all text is encoded/decoded as UTF-8. This can be changed via the {@code charsetName} constructor
    078 * argument.
    079 */
    080public class TextLine extends Scheme<Configuration, RecordReader, OutputCollector, Object[], Object[]> implements FileFormat
    081  {
    082  public enum Compress
    083    {
    084      DEFAULT, ENABLE, DISABLE
    085    }
    086
    087  public static final String DEFAULT_CHARSET = "UTF-8";
    088
    089  /** Field serialVersionUID */
    090  private static final long serialVersionUID = 1L;
    091  /** Field DEFAULT_SOURCE_FIELDS */
    092  public static final Fields DEFAULT_SOURCE_FIELDS = new Fields( "offset", "line" ).applyTypes( Long.TYPE, String.class );
    093
    094  /** Field sinkCompression */
    095  Compress sinkCompression = Compress.DISABLE;
    096
    097  String charsetName = DEFAULT_CHARSET;
    098
    099  /**
    100   * Creates a new TextLine instance that sources "offset" and "line" fields, and sinks all incoming fields, where
    101   * "offset" is the byte offset in the input file.
    102   */
    103  public TextLine()
    104    {
    105    super( DEFAULT_SOURCE_FIELDS );
    106    }
    107
    108  /**
    109   * Creates a new TextLine instance that sources "offset" and "line" fields, and sinks all incoming fields, where
    110   * "offset" is the byte offset in the input file.
    111   *
    112   * @param numSinkParts of type int
    113   */
    114  @ConstructorProperties({"numSinkParts"})
    115  public TextLine( int numSinkParts )
    116    {
    117    super( DEFAULT_SOURCE_FIELDS, numSinkParts );
    118    }
    119
    120  /**
    121   * Creates a new TextLine instance that sources "offset" and "line" fields, and sinks all incoming fields, where
    122   * "offset" is the byte offset in the input file.
    123   *
    124   * @param sinkCompression of type Compress
    125   */
    126  @ConstructorProperties({"sinkCompression"})
    127  public TextLine( Compress sinkCompression )
    128    {
    129    super( DEFAULT_SOURCE_FIELDS );
    130
    131    setSinkCompression( sinkCompression );
    132    }
    133
    134  /**
    135   * Creates a new TextLine instance. If sourceFields has one field, only the text line will be returned in the
    136   * subsequent tuples.
    137   *
    138   * @param sourceFields the source fields for this scheme
    139   * @param sinkFields   the sink fields for this scheme
    140   */
    141  @ConstructorProperties({"sourceFields", "sinkFields"})
    142  public TextLine( Fields sourceFields, Fields sinkFields )
    143    {
    144    super( sourceFields, sinkFields );
    145
    146    verify( sourceFields );
    147    }
    148
    149  /**
    150   * Creates a new TextLine instance. If sourceFields has one field, only the text line will be returned in the
    151   * subsequent tuples.
    152   *
    153   * @param sourceFields the source fields for this scheme
    154   * @param sinkFields   the sink fields for this scheme
    155   * @param charsetName  of type String
    156   */
    157  @ConstructorProperties({"sourceFields", "sinkFields", "charsetName"})
    158  public TextLine( Fields sourceFields, Fields sinkFields, String charsetName )
    159    {
    160    super( sourceFields, sinkFields );
    161
    162    // throws an exception if not found
    163    setCharsetName( charsetName );
    164
    165    verify( sourceFields );
    166    }
    167
    168  /**
    169   * Creates a new TextLine instance. If sourceFields has one field, only the text line will be returned in the
    170   * subsequent tuples.
    171   *
    172   * @param sourceFields the source fields for this scheme
    173   * @param sinkFields   the sink fields for this scheme
    174   * @param numSinkParts of type int
    175   */
    176  @ConstructorProperties({"sourceFields", "sinkFields", "numSinkParts"})
    177  public TextLine( Fields sourceFields, Fields sinkFields, int numSinkParts )
    178    {
    179    super( sourceFields, sinkFields, numSinkParts );
    180
    181    verify( sourceFields );
    182    }
    183
    184  /**
    185   * Constructor TextLine creates a new TextLine instance. If sourceFields has one field, only the text line will be returned in the
    186   * subsequent tuples.
    187   *
    188   * @param sourceFields    of type Fields
    189   * @param sinkFields      of type Fields
    190   * @param sinkCompression of type Compress
    191   */
    192  @ConstructorProperties({"sourceFields", "sinkFields", "sinkCompression"})
    193  public TextLine( Fields sourceFields, Fields sinkFields, Compress sinkCompression )
    194    {
    195    super( sourceFields, sinkFields );
    196
    197    setSinkCompression( sinkCompression );
    198
    199    verify( sourceFields );
    200    }
    201
    202  /**
    203   * Constructor TextLine creates a new TextLine instance. If sourceFields has one field, only the text line will be returned in the
    204   * subsequent tuples.
    205   *
    206   * @param sourceFields    of type Fields
    207   * @param sinkFields      of type Fields
    208   * @param sinkCompression of type Compress
    209   * @param charsetName     of type String
    210   */
    211  @ConstructorProperties({"sourceFields", "sinkFields", "sinkCompression", "charsetName"})
    212  public TextLine( Fields sourceFields, Fields sinkFields, Compress sinkCompression, String charsetName )
    213    {
    214    super( sourceFields, sinkFields );
    215
    216    setSinkCompression( sinkCompression );
    217
    218    // throws an exception if not found
    219    setCharsetName( charsetName );
    220
    221    verify( sourceFields );
    222    }
    223
    224  /**
    225   * Constructor TextLine creates a new TextLine instance. If sourceFields has one field, only the text line will be returned in the
    226   * subsequent tuples.
    227   *
    228   * @param sourceFields    of type Fields
    229   * @param sinkFields      of type Fields
    230   * @param sinkCompression of type Compress
    231   * @param numSinkParts    of type int
    232   */
    233  @ConstructorProperties({"sourceFields", "sinkFields", "sinkCompression", "numSinkParts"})
    234  public TextLine( Fields sourceFields, Fields sinkFields, Compress sinkCompression, int numSinkParts )
    235    {
    236    super( sourceFields, sinkFields, numSinkParts );
    237
    238    setSinkCompression( sinkCompression );
    239
    240    verify( sourceFields );
    241    }
    242
    243  /**
    244   * Constructor TextLine creates a new TextLine instance. If sourceFields has one field, only the text line will be returned in the
    245   * subsequent tuples.
    246   *
    247   * @param sourceFields    of type Fields
    248   * @param sinkFields      of type Fields
    249   * @param sinkCompression of type Compress
    250   * @param numSinkParts    of type int
    251   * @param charsetName     of type String
    252   */
    253  @ConstructorProperties({"sourceFields", "sinkFields", "sinkCompression", "numSinkParts", "charsetName"})
    254  public TextLine( Fields sourceFields, Fields sinkFields, Compress sinkCompression, int numSinkParts, String charsetName )
    255    {
    256    super( sourceFields, sinkFields, numSinkParts );
    257
    258    setSinkCompression( sinkCompression );
    259
    260    // throws an exception if not found
    261    setCharsetName( charsetName );
    262
    263    verify( sourceFields );
    264    }
    265
    266  /**
    267   * Creates a new TextLine instance. If sourceFields has one field, only the text line will be returned in the
    268   * subsequent tuples.
    269   *
    270   * @param sourceFields the source fields for this scheme
    271   */
    272  @ConstructorProperties({"sourceFields"})
    273  public TextLine( Fields sourceFields )
    274    {
    275    super( sourceFields );
    276
    277    verify( sourceFields );
    278    }
    279
    280  /**
    281   * Creates a new TextLine instance. If sourceFields has one field, only the text line will be returned in the
    282   * subsequent tuples.
    283   *
    284   * @param sourceFields the source fields for this scheme
    285   * @param charsetName  of type String
    286   */
    287  @ConstructorProperties({"sourceFields", "charsetName"})
    288  public TextLine( Fields sourceFields, String charsetName )
    289    {
    290    super( sourceFields );
    291
    292    // throws an exception if not found
    293    setCharsetName( charsetName );
    294
    295    verify( sourceFields );
    296    }
    297
    298  /**
    299   * Creates a new TextLine instance. If sourceFields has one field, only the text line will be returned in the
    300   * subsequent tuples. The resulting data set will have numSinkParts.
    301   *
    302   * @param sourceFields the source fields for this scheme
    303   * @param numSinkParts of type int
    304   */
    305  @ConstructorProperties({"sourceFields", "numSinkParts"})
    306  public TextLine( Fields sourceFields, int numSinkParts )
    307    {
    308    super( sourceFields, numSinkParts );
    309
    310    verify( sourceFields );
    311    }
    312
    313  protected void setCharsetName( String charsetName )
    314    {
    315    if( charsetName != null )
    316      this.charsetName = charsetName;
    317
    318    Charset.forName( this.charsetName );
    319    }
    320
    321  @Property(name = "charset", visibility = Visibility.PUBLIC)
    322  @PropertyDescription(value = "character set used in this scheme.")
    323  public String getCharsetName()
    324    {
    325    return charsetName;
    326    }
    327
    328  protected void verify( Fields sourceFields )
    329    {
    330    if( sourceFields.size() < 1 || sourceFields.size() > 2 )
    331      throw new IllegalArgumentException( "this scheme requires either one or two source fields, given [" + sourceFields + "]" );
    332    }
    333
    334  /**
    335   * Method getSinkCompression returns the sinkCompression of this TextLine object.
    336   *
    337   * @return the sinkCompression (type Compress) of this TextLine object.
    338   */
    339  @Property(name = "sinkCompression", visibility = Visibility.PUBLIC)
    340  @PropertyDescription(value = "The compression of the scheme when used in a sink.")
    341  public Compress getSinkCompression()
    342    {
    343    return sinkCompression;
    344    }
    345
    346  /**
    347   * Method setSinkCompression sets the sinkCompression of this TextLine object. If null, compression will remain disabled.
    348   *
    349   * @param sinkCompression the sinkCompression of this TextLine object.
    350   */
    351  public void setSinkCompression( Compress sinkCompression )
    352    {
    353    if( sinkCompression != null ) // leave disabled if null
    354      this.sinkCompression = sinkCompression;
    355    }
    356
    357  @Override
    358  public void sourceConfInit( FlowProcess<? extends Configuration> flowProcess, Tap<Configuration, RecordReader, OutputCollector> tap, Configuration conf )
    359    {
    360    if( hasZippedFiles( FileInputFormat.getInputPaths( asJobConfInstance( conf ) ) ) )
    361      throw new IllegalStateException( "cannot read zip files: " + Arrays.toString( FileInputFormat.getInputPaths( asJobConfInstance( conf ) ) ) );
    362
    363    conf.setBoolean( "mapred.mapper.new-api", false );
    364    conf.setClass( "mapred.input.format.class", TextInputFormat.class, InputFormat.class );
    365    }
    366
    367  private boolean hasZippedFiles( Path[] paths )
    368    {
    369    if( paths == null || paths.length == 0 )
    370      return false;
    371
    372    boolean isZipped = paths[ 0 ].getName().endsWith( ".zip" );
    373
    374    for( int i = 1; i < paths.length; i++ )
    375      {
    376      if( isZipped != paths[ i ].getName().endsWith( ".zip" ) )
    377        throw new IllegalStateException( "cannot mix zipped and upzipped files" );
    378      }
    379
    380    return isZipped;
    381    }
    382
    383  @Override
    384  public void presentSourceFields( FlowProcess<? extends Configuration> flowProcess, Tap tap, Fields fields )
    385    {
    386    // do nothing to change TextLine state
    387    }
    388
    389  @Override
    390  public void presentSinkFields( FlowProcess<? extends Configuration> flowProcess, Tap tap, Fields fields )
    391    {
    392    // do nothing to change TextLine state
    393    }
    394
    395  @Override
    396  public void sinkConfInit( FlowProcess<? extends Configuration> flowProcess, Tap<Configuration, RecordReader, OutputCollector> tap, Configuration conf )
    397    {
    398    if( tap.getFullIdentifier( conf ).endsWith( ".zip" ) )
    399      throw new IllegalStateException( "cannot write zip files: " + HadoopUtil.getOutputPath( conf ) );
    400
    401    conf.setBoolean( "mapred.mapper.new-api", false );
    402
    403    if( getSinkCompression() == Compress.DISABLE )
    404      conf.setBoolean( "mapred.output.compress", false );
    405    else if( getSinkCompression() == Compress.ENABLE )
    406      conf.setBoolean( "mapred.output.compress", true );
    407
    408    conf.setClass( "mapred.output.key.class", Text.class, Object.class );
    409    conf.setClass( "mapred.output.value.class", Text.class, Object.class );
    410    conf.setClass( "mapred.output.format.class", TextOutputFormat.class, OutputFormat.class );
    411    }
    412
    413  @Override
    414  public void sourcePrepare( FlowProcess<? extends Configuration> flowProcess, SourceCall<Object[], RecordReader> sourceCall )
    415    {
    416    if( sourceCall.getContext() == null )
    417      sourceCall.setContext( new Object[ 3 ] );
    418
    419    sourceCall.getContext()[ 0 ] = sourceCall.getInput().createKey();
    420    sourceCall.getContext()[ 1 ] = sourceCall.getInput().createValue();
    421    sourceCall.getContext()[ 2 ] = Charset.forName( charsetName );
    422    }
    423
    424  @Override
    425  public boolean source( FlowProcess<? extends Configuration> flowProcess, SourceCall<Object[], RecordReader> sourceCall ) throws IOException
    426    {
    427    if( !sourceReadInput( sourceCall ) )
    428      return false;
    429
    430    sourceHandleInput( sourceCall );
    431
    432    return true;
    433    }
    434
    435  private boolean sourceReadInput( SourceCall<Object[], RecordReader> sourceCall ) throws IOException
    436    {
    437    Object[] context = sourceCall.getContext();
    438
    439    return sourceCall.getInput().next( context[ 0 ], context[ 1 ] );
    440    }
    441
    442  protected void sourceHandleInput( SourceCall<Object[], RecordReader> sourceCall )
    443    {
    444    TupleEntry result = sourceCall.getIncomingEntry();
    445
    446    int index = 0;
    447    Object[] context = sourceCall.getContext();
    448
    449    // coerce into canonical forms
    450    if( getSourceFields().size() == 2 )
    451      result.setLong( index++, ( (LongWritable) context[ 0 ] ).get() );
    452
    453    result.setString( index, makeEncodedString( context ) );
    454    }
    455
    456  protected String makeEncodedString( Object[] context )
    457    {
    458    Text text = (Text) context[ 1 ];
    459    return new String( text.getBytes(), 0, text.getLength(), (Charset) context[ 2 ] );
    460    }
    461
    462  @Override
    463  public void sourceCleanup( FlowProcess<? extends Configuration> flowProcess, SourceCall<Object[], RecordReader> sourceCall )
    464    {
    465    sourceCall.setContext( null );
    466    }
    467
    468  @Override
    469  public void sinkPrepare( FlowProcess<? extends Configuration> flowProcess, SinkCall<Object[], OutputCollector> sinkCall ) throws IOException
    470    {
    471    sinkCall.setContext( new Object[ 2 ] );
    472
    473    sinkCall.getContext()[ 0 ] = new Text();
    474    sinkCall.getContext()[ 1 ] = Charset.forName( charsetName );
    475    }
    476
    477  @Override
    478  public void sink( FlowProcess<? extends Configuration> flowProcess, SinkCall<Object[], OutputCollector> sinkCall ) throws IOException
    479    {
    480    Text text = (Text) sinkCall.getContext()[ 0 ];
    481    Charset charset = (Charset) sinkCall.getContext()[ 1 ];
    482    String line = sinkCall.getOutgoingEntry().getTuple().toString();
    483
    484    text.set( line.getBytes( charset ) );
    485
    486    // it's ok to use NULL here so the collector does not write anything
    487    sinkCall.getOutput().collect( null, text );
    488    }
    489
    490  @Override
    491  public String getExtension()
    492    {
    493    return "txt";
    494    }
    495  }