    001/*
    002 * Copyright (c) 2007-2017 Xplenty, Inc. All Rights Reserved.
    003 *
    004 * Project and contact information: http://www.cascading.org/
    005 *
    006 * This file is part of the Cascading project.
    007 *
    008 * Licensed under the Apache License, Version 2.0 (the "License");
    009 * you may not use this file except in compliance with the License.
    010 * You may obtain a copy of the License at
    011 *
    012 *     http://www.apache.org/licenses/LICENSE-2.0
    013 *
    014 * Unless required by applicable law or agreed to in writing, software
    015 * distributed under the License is distributed on an "AS IS" BASIS,
    016 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    017 * See the License for the specific language governing permissions and
    018 * limitations under the License.
    019 */
    020
    021package cascading.tap;
    022
    023import java.beans.ConstructorProperties;
    024import java.io.IOException;
    025import java.util.ArrayList;
    026import java.util.Arrays;
    027import java.util.HashSet;
    028import java.util.Iterator;
    029import java.util.List;
    030import java.util.Map;
    031import java.util.Set;
    032
    033import cascading.flow.FlowProcess;
    034import cascading.scheme.NullScheme;
    035import cascading.scheme.Scheme;
    036import cascading.tuple.Fields;
    037import cascading.tuple.TupleEntry;
    038import cascading.tuple.TupleEntryCollector;
    039import cascading.util.Util;
    040import org.slf4j.Logger;
    041import org.slf4j.LoggerFactory;
    042
    043/**
    044 * Class MultiSinkTap is both a {@link cascading.tap.CompositeTap} and {@link cascading.tap.SinkTap} that can write to
    045 * multiple child {@link cascading.tap.Tap} instances simultaneously.
    046 * <p/>
    047 * It is the counterpart to {@link cascading.tap.MultiSourceTap}.
    048 * <p/>
    049 * Note all child Tap instances may or may not have the same declared Fields. In the case they do not, all
    050 * sink fields will be merged into a single Fields instance via {@link Fields#merge(cascading.tuple.Fields...)}.
    051 */
    052public class MultiSinkTap<Child extends Tap, Config, Output> extends SinkTap<Config, Output> implements CompositeTap<Child>
    053  {
    054  /** Field LOG */
    055  private static final Logger LOG = LoggerFactory.getLogger( MultiSinkTap.class );
    056
    057  /** Field taps */
    058  private final Child[] taps;
    059  /** Field tempPath */
    060  private final String tempPath = "__multisink_placeholder_" + Util.createUniqueID();
    061  /** Field childConfigs */
    062  private List<Map<String, String>> childConfigs;
    063
    064  private class MultiSinkCollector extends TupleEntryCollector
    065    {
    066    TupleEntryCollector[] collectors;
    067
    068    public <C extends Config> MultiSinkCollector( FlowProcess<C> flowProcess, Tap<Config, ?, ?>... taps ) throws IOException
    069      {
    070      super( Fields.asDeclaration( getSinkFields() ) );
    071
    072      collectors = new TupleEntryCollector[ taps.length ];
    073
    074      C conf = flowProcess.getConfigCopy();
    075
    076      for( int i = 0; i < taps.length; i++ )
    077        {
    078        C mergedConf = childConfigs == null ? conf : flowProcess.mergeMapIntoConfig( conf, childConfigs.get( i ) );
    079        Tap<Config, ?, ?> tap = taps[ i ];
    080        LOG.info( "opening for write: {}", tap.toString() );
    081
    082        collectors[ i ] = tap.openForWrite( flowProcess.copyWith( mergedConf ), null );
    083        }
    084      }
    085
    086    protected void collect( TupleEntry tupleEntry ) throws IOException
    087      {
    088      for( int i = 0; i < taps.length; i++ )
    089        collectors[ i ].add( tupleEntry );
    090      }
    091
    092    @Override
    093    public void close()
    094      {
    095      super.close();
    096
    097      try
    098        {
    099        for( TupleEntryCollector collector : collectors )
    100          {
    101          try
    102            {
    103            collector.close();
    104            }
    105          catch( Exception exception )
    106            {
    107            LOG.warn( "exception closing TupleEntryCollector", exception );
    108            }
    109          }
    110        }
    111      finally
    112        {
    113        collectors = null;
    114        }
    115      }
    116    }
    117
    118  /**
    119   * Constructor MultiSinkTap creates a new MultiSinkTap instance.
    120   *
    121   * @param taps of type Tap...
    122   */
    123  @ConstructorProperties({"taps"})
    124  public MultiSinkTap( Child... taps )
    125    {
    126    this.taps = taps;
    127    }
    128
    129  protected Child[] getTaps()
    130    {
    131    return taps;
    132    }
    133
    134  @Override
    135  public Iterator<Child> getChildTaps()
    136    {
    137    return Arrays.asList( getTaps() ).iterator();
    138    }
    139
    140  @Override
    141  public long getNumChildTaps()
    142    {
    143    return getTaps().length;
    144    }
    145
    146  @Override
    147  public String getIdentifier()
    148    {
    149    return tempPath;
    150    }
    151
    152  @Override
    153  public void presentSinkFields( FlowProcess<? extends Config> flowProcess, Fields fields )
    154    {
    155    for( Tap child : getTaps() )
    156      child.presentSinkFields( flowProcess, fields );
    157    }
    158
    159  @Override
    160  public TupleEntryCollector openForWrite( FlowProcess<? extends Config> flowProcess, Output output ) throws IOException
    161    {
    162    return new MultiSinkCollector( flowProcess, getTaps() );
    163    }
    164
    165  @Override
    166  public void sinkConfInit( FlowProcess<? extends Config> flowProcess, Config conf )
    167    {
    168    bridge( flowProcess, conf );
    169    }
    170
    171  private void bridge( FlowProcess flowProcess, Object conf )
    172    {
    173    childConfigs = new ArrayList<>();
    174
    175    for( int i = 0; i < getTaps().length; i++ )
    176      {
    177      Tap tap = getTaps()[ i ];
    178      Object newConfig = flowProcess.copyConfig( conf );
    179
    180      tap.sinkConfInit( flowProcess, newConfig );
    181
    182      childConfigs.add( flowProcess.diffConfigIntoMap( conf, newConfig ) );
    183      }
    184    }
    185
    186  @Override
    187  public boolean createResource( Config conf ) throws IOException
    188    {
    189    for( Tap tap : getTaps() )
    190      {
    191      if( !tap.createResource( conf ) )
    192        return false;
    193      }
    194
    195    return true;
    196    }
    197
    198  @Override
    199  public boolean deleteResource( Config conf ) throws IOException
    200    {
    201    for( Tap tap : getTaps() )
    202      {
    203      if( !tap.deleteResource( conf ) )
    204        return false;
    205      }
    206
    207    return true;
    208    }
    209
    210  @Override
    211  public boolean commitResource( Config conf ) throws IOException
    212    {
    213    for( Tap tap : getTaps() )
    214      {
    215      if( !tap.commitResource( conf ) )
    216        return false;
    217      }
    218
    219    return true;
    220    }
    221
    222  @Override
    223  public boolean rollbackResource( Config conf ) throws IOException
    224    {
    225    for( Tap tap : getTaps() )
    226      {
    227      if( !tap.rollbackResource( conf ) )
    228        return false;
    229      }
    230
    231    return true;
    232    }
    233
    234  @Override
    235  public boolean resourceExists( Config conf ) throws IOException
    236    {
    237    for( Tap tap : getTaps() )
    238      {
    239      if( !tap.resourceExists( conf ) )
    240        return false;
    241      }
    242
    243    return true;
    244    }
    245
    246  @Override
    247  public long getModifiedTime( Config conf ) throws IOException
    248    {
    249    long modified = getTaps()[ 0 ].getModifiedTime( conf );
    250
    251    for( int i = 1; i < getTaps().length; i++ )
    252      modified = Math.max( getTaps()[ i ].getModifiedTime( conf ), modified );
    253
    254    return modified;
    255    }
    256
    257  @Override
    258  public Scheme getScheme()
    259    {
    260    if( super.getScheme() != null )
    261      return super.getScheme();
    262
    263    Set<Fields> fields = new HashSet<Fields>();
    264
    265    for( Tap child : getTaps() )
    266      fields.add( child.getSinkFields() );
    267
    268    // if all schemes have the same sink fields, the just use the scheme
    269    if( fields.size() == 1 )
    270      {
    271      setScheme( getTaps()[ 0 ].getScheme() );
    272      return super.getScheme();
    273      }
    274
    275    Fields allFields = Fields.merge( fields.toArray( new Fields[ fields.size() ] ) );
    276
    277    setScheme( new NullScheme( allFields, allFields ) );
    278
    279    return super.getScheme();
    280    }
    281
    282  @Override
    283  public String toString()
    284    {
    285    return "MultiSinkTap[" + ( taps == null ? "none" : Arrays.asList( taps ) ) + ']';
    286    }
    287
    288  @Override
    289  public boolean equals( Object o )
    290    {
    291    if( this == o )
    292      return true;
    293    if( !( o instanceof MultiSinkTap ) )
    294      return false;
    295    if( !super.equals( o ) )
    296      return false;
    297
    298    MultiSinkTap that = (MultiSinkTap) o;
    299
    300    if( !Arrays.equals( taps, that.taps ) )
    301      return false;
    302
    303    return true;
    304    }
    305
    306  @Override
    307  public int hashCode()
    308    {
    309    int result = super.hashCode();
    310    result = 31 * result + ( taps != null ? Arrays.hashCode( taps ) : 0 );
    311    return result;
    312    }
    313  }