    001/*
    002 * Copyright (c) 2016-2017 Chris K Wensel <chris@wensel.net>. All Rights Reserved.
    003 * Copyright (c) 2007-2017 Xplenty, Inc. All Rights Reserved.
    004 *
    005 * Project and contact information: http://www.cascading.org/
    006 *
    007 * This file is part of the Cascading project.
    008 *
    009 * Licensed under the Apache License, Version 2.0 (the "License");
    010 * you may not use this file except in compliance with the License.
    011 * You may obtain a copy of the License at
    012 *
    013 *     http://www.apache.org/licenses/LICENSE-2.0
    014 *
    015 * Unless required by applicable law or agreed to in writing, software
    016 * distributed under the License is distributed on an "AS IS" BASIS,
    017 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    018 * See the License for the specific language governing permissions and
    019 * limitations under the License.
    020 */
    021
    022package cascading.tuple;
    023
    024import java.io.Closeable;
    025import java.io.IOException;
    026import java.util.Collections;
    027import java.util.Set;
    028
    029import cascading.flow.FlowProcess;
    030import cascading.scheme.ConcreteCall;
    031import cascading.scheme.Scheme;
    032import cascading.tap.Tap;
    033import cascading.util.CloseableIterator;
    034import cascading.util.SingleCloseableInputIterator;
    035import cascading.util.Util;
    036import org.slf4j.Logger;
    037import org.slf4j.LoggerFactory;
    038
    039/**
    040 * Class TupleEntrySchemeIterator is a helper class for wrapping a {@link Scheme} instance, calling
    041 * {@link Scheme#source(cascading.flow.FlowProcess, cascading.scheme.SourceCall)} on every call to
    042 * {@link #next()}. The behavior can be controlled via properties defined in {@link TupleEntrySchemeIteratorProps}.
    043 * <p/>
    044 * Use this class inside a custom {@link cascading.tap.Tap} when overriding the
    045 * {@link cascading.tap.Tap#openForRead(cascading.flow.FlowProcess)} method.
    046 */
    047public class TupleEntrySchemeIterator<Config, Input> extends TupleEntryIterator
    048  {
    049  /** Field LOG */
    050  private static final Logger LOG = LoggerFactory.getLogger( TupleEntrySchemeIterator.class );
    051
    052  private final FlowProcess<? extends Config> flowProcess;
    053  private final Scheme scheme;
    054  private final CloseableIterator<Input> inputIterator;
    055  private final Set<Class<? extends Exception>> permittedExceptions;
    056  private ConcreteCall sourceCall;
    057
    058  private String loggableIdentifier;
    059  private boolean isComplete = false;
    060  private boolean hasWaiting = false;
    061  private TupleException currentException;
    062
    063  @Deprecated
    064  public TupleEntrySchemeIterator( FlowProcess<? extends Config> flowProcess, Scheme scheme, Input input )
    065    {
    066    this( flowProcess, scheme, input, null );
    067    }
    068
    069  @Deprecated
    070  public TupleEntrySchemeIterator( FlowProcess<? extends Config> flowProcess, Scheme scheme, Input input, String loggableIdentifier )
    071    {
    072    this( flowProcess, scheme, (CloseableIterator<Input>) new SingleCloseableInputIterator( (Closeable) input ), loggableIdentifier );
    073    }
    074
    075  @Deprecated
    076  public TupleEntrySchemeIterator( FlowProcess<? extends Config> flowProcess, Scheme scheme, CloseableIterator<Input> inputIterator )
    077    {
    078    this( flowProcess, scheme, inputIterator, null );
    079    }
    080
    081  @Deprecated
    082  public TupleEntrySchemeIterator( FlowProcess<? extends Config> flowProcess, Scheme scheme, CloseableIterator<Input> inputIterator, String loggableIdentifier )
    083    {
    084    this( flowProcess, null, scheme, inputIterator, loggableIdentifier );
    085    }
    086
    087  public TupleEntrySchemeIterator( FlowProcess<? extends Config> flowProcess, Tap tap, Scheme scheme, Input input )
    088    {
    089    this( flowProcess, tap, scheme, input, null );
    090    }
    091
    092  public TupleEntrySchemeIterator( FlowProcess<? extends Config> flowProcess, Tap tap, Scheme scheme, Input input, String loggableIdentifier )
    093    {
    094    this( flowProcess, tap, scheme, (CloseableIterator<Input>) new SingleCloseableInputIterator( (Closeable) input ), loggableIdentifier );
    095    }
    096
    097  public TupleEntrySchemeIterator( FlowProcess<? extends Config> flowProcess, Tap tap, Scheme scheme, CloseableIterator<Input> inputIterator )
    098    {
    099    this( flowProcess, tap, scheme, inputIterator, null );
    100    }
    101
    102  public TupleEntrySchemeIterator( FlowProcess<? extends Config> flowProcess, Tap tap, Scheme scheme, CloseableIterator<Input> inputIterator, String loggableIdentifier )
    103    {
    104    super( scheme.getSourceFields() );
    105    this.flowProcess = flowProcess;
    106    this.scheme = scheme;
    107    this.inputIterator = inputIterator;
    108    this.loggableIdentifier = loggableIdentifier;
    109
    110    Object permittedExceptions = flowProcess.getProperty( TupleEntrySchemeIteratorProps.PERMITTED_EXCEPTIONS );
    111
    112    if( permittedExceptions != null )
    113      this.permittedExceptions = Util.asClasses( permittedExceptions.toString(), "unable to load permitted exception class" );
    114    else
    115      this.permittedExceptions = Collections.emptySet();
    116
    117    if( this.loggableIdentifier == null || this.loggableIdentifier.isEmpty() )
    118      this.loggableIdentifier = "'unknown'";
    119
    120    if( !inputIterator.hasNext() )
    121      {
    122      isComplete = true;
    123      return;
    124      }
    125
    126    sourceCall = new ConcreteCall();
    127
    128    sourceCall.setTap( tap );
    129    sourceCall.setIncomingEntry( getTupleEntry() );
    130    sourceCall.setInput( wrapInput( inputIterator.next() ) );
    131
    132    try
    133      {
    134      this.scheme.sourcePrepare( flowProcess, sourceCall );
    135      }
    136    catch( IOException exception )
    137      {
    138      throw new TupleException( "unable to prepare source for input identifier: " + this.loggableIdentifier, exception );
    139      }
    140    }
    141
    142  protected FlowProcess<? extends Config> getFlowProcess()
    143    {
    144    return flowProcess;
    145    }
    146
    147  protected Input wrapInput( Input input )
    148    {
    149    return input;
    150    }
    151
    152  @Override
    153  public boolean hasNext()
    154    {
    155    if( currentException != null )
    156      return true;
    157
    158    if( isComplete )
    159      return false;
    160
    161    if( hasWaiting )
    162      return true;
    163
    164    try
    165      {
    166      getNext();
    167      }
    168    catch( Exception exception )
    169      {
    170      if( loggableIdentifier == null || loggableIdentifier.isEmpty() )
    171        loggableIdentifier = "'unknown'";
    172
    173      if( permittedExceptions.contains( exception.getClass() ) )
    174        {
    175        LOG.warn( "Caught permitted exception while reading {}", loggableIdentifier, exception );
    176        return false;
    177        }
    178
    179      currentException = new TupleException( "unable to read from input identifier: " + loggableIdentifier, exception );
    180
    181      return true;
    182      }
    183
    184    if( !hasWaiting )
    185      isComplete = true;
    186
    187    return !isComplete;
    188    }
    189
    190  private TupleEntry getNext() throws IOException
    191    {
    192    Tuples.asModifiable( sourceCall.getIncomingEntry().getTuple() );
    193    hasWaiting = scheme.source( flowProcess, sourceCall );
    194
    195    while( !hasWaiting && inputIterator.hasNext() )
    196      {
    197      sourceCall.setInput( wrapInput( inputIterator.next() ) );
    198
    199      try
    200        {
    201        scheme.sourceRePrepare( flowProcess, sourceCall );
    202        }
    203      catch( IOException exception )
    204        {
    205        throw new TupleException( "unable to prepare source for input identifier: " + loggableIdentifier, exception );
    206        }
    207
    208      Tuples.asModifiable( sourceCall.getIncomingEntry().getTuple() );
    209      hasWaiting = scheme.source( flowProcess, sourceCall );
    210      }
    211
    212    return getTupleEntry();
    213    }
    214
    215  @Override
    216  public TupleEntry next()
    217    {
    218    try
    219      {
    220      if( currentException != null )
    221        throw currentException;
    222      }
    223    finally
    224      {
    225      currentException = null; // data may be trapped
    226      }
    227
    228    if( isComplete )
    229      throw new IllegalStateException( "no next element" );
    230
    231    try
    232      {
    233      if( hasWaiting )
    234        return getTupleEntry();
    235
    236      return getNext();
    237      }
    238    catch( Exception exception )
    239      {
    240      throw new TupleException( "unable to source from input identifier: " + loggableIdentifier, exception );
    241      }
    242    finally
    243      {
    244      hasWaiting = false;
    245      }
    246    }
    247
    248  @Override
    249  public void remove()
    250    {
    251    throw new UnsupportedOperationException( "may not remove elements from this iterator" );
    252    }
    253
    254  @Override
    255  public void close() throws IOException
    256    {
    257    try
    258      {
    259      if( sourceCall != null )
    260        scheme.sourceCleanup( flowProcess, sourceCall );
    261      }
    262    finally
    263      {
    264      inputIterator.close();
    265      }
    266    }
    267  }