    001/*
    002 * Copyright (c) 2007-2017 Xplenty, Inc. All Rights Reserved.
    003 *
    004 * Project and contact information: http://www.cascading.org/
    005 *
    006 * This file is part of the Cascading project.
    007 *
    008 * Licensed under the Apache License, Version 2.0 (the "License");
    009 * you may not use this file except in compliance with the License.
    010 * You may obtain a copy of the License at
    011 *
    012 *     http://www.apache.org/licenses/LICENSE-2.0
    013 *
    014 * Unless required by applicable law or agreed to in writing, software
    015 * distributed under the License is distributed on an "AS IS" BASIS,
    016 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    017 * See the License for the specific language governing permissions and
    018 * limitations under the License.
    019 */
    020
    021package cascading.flow;
    022
    023import java.io.IOException;
    024import java.util.ArrayList;
    025import java.util.Collection;
    026import java.util.Collections;
    027import java.util.Date;
    028import java.util.HashMap;
    029import java.util.HashSet;
    030import java.util.Iterator;
    031import java.util.LinkedHashMap;
    032import java.util.LinkedList;
    033import java.util.List;
    034import java.util.Map;
    035import java.util.Properties;
    036import java.util.Set;
    037import java.util.concurrent.Callable;
    038import java.util.concurrent.ExecutionException;
    039import java.util.concurrent.Future;
    040import java.util.concurrent.TimeUnit;
    041import java.util.concurrent.locks.ReentrantLock;
    042
    043import cascading.CascadingException;
    044import cascading.cascade.Cascade;
    045import cascading.flow.planner.BaseFlowNode;
    046import cascading.flow.planner.BaseFlowStep;
    047import cascading.flow.planner.FlowStepJob;
    048import cascading.flow.planner.PlannerInfo;
    049import cascading.flow.planner.PlatformInfo;
    050import cascading.flow.planner.graph.ElementGraphs;
    051import cascading.flow.planner.graph.FlowElementGraph;
    052import cascading.flow.planner.process.FlowStepGraph;
    053import cascading.flow.planner.process.ProcessGraphs;
    054import cascading.management.CascadingServices;
    055import cascading.management.UnitOfWorkExecutorStrategy;
    056import cascading.management.UnitOfWorkSpawnStrategy;
    057import cascading.management.state.ClientState;
    058import cascading.property.AppProps;
    059import cascading.property.PropertyUtil;
    060import cascading.stats.FlowStats;
    061import cascading.tap.Tap;
    062import cascading.tuple.Fields;
    063import cascading.tuple.TupleEntryCollector;
    064import cascading.tuple.TupleEntryIterator;
    065import cascading.util.ProcessLogger;
    066import cascading.util.ShutdownUtil;
    067import cascading.util.Update;
    068import cascading.util.Util;
    069import cascading.util.Version;
    070import org.slf4j.Logger;
    071import org.slf4j.LoggerFactory;
    072import riffle.process.DependencyIncoming;
    073import riffle.process.DependencyOutgoing;
    074import riffle.process.ProcessCleanup;
    075import riffle.process.ProcessComplete;
    076import riffle.process.ProcessPrepare;
    077import riffle.process.ProcessStart;
    078import riffle.process.ProcessStop;
    079
    080import static cascading.util.Util.formatDurationFromMillis;
    081
    082@riffle.process.Process
    083public abstract class BaseFlow<Config> implements Flow<Config>, ProcessLogger
    084  {
    085  /** Field LOG */
    086  private static final Logger LOG = LoggerFactory.getLogger( Flow.class ); // wrapped by ProcessLogger interface methods
    087  private static final int LOG_FLOW_NAME_MAX = 25;
    088
    089  private PlannerInfo plannerInfo = PlannerInfo.NULL;
    090  protected PlatformInfo platformInfo = PlatformInfo.NULL;
    091
    092  /** Field id */
    093  private String id;
    094  /** Field name */
    095  private String name;
    096  /** Fields runID */
    097  private String runID;
    098  /** Fields classpath */
    099  private List<String> classPath; // may remain null
    100  /** Field tags */
    101  private String tags;
    102  /** Field listeners */
    103  private List<SafeFlowListener> listeners;
    104  /** Field skipStrategy */
    105  private FlowSkipStrategy flowSkipStrategy = new FlowSkipIfSinkNotStale();
    106  /** Field flowStats */
    107  protected FlowStats flowStats; // don't use a listener to set values
    108  /** Field sources */
    109  protected Map<String, Tap> sources = Collections.emptyMap();
    110  /** Field sinks */
    111  protected Map<String, Tap> sinks = Collections.emptyMap();
    112  /** Field traps */
    113  private Map<String, Tap> traps = Collections.emptyMap();
    114  /** Field checkpoints */
    115  private Map<String, Tap> checkpoints = Collections.emptyMap();
    116  /** Field stopJobsOnExit */
    117  protected boolean stopJobsOnExit = true;
    118  /** Field submitPriority */
    119  private int submitPriority = 5;
    120
    121  /** Field stepGraph */
    122  protected FlowStepGraph flowStepGraph;
    123  /** Field thread */
    124  protected transient Thread thread;
    125  /** Field throwable */
    126  protected Throwable throwable;
    127  /** Field stop */
    128  protected volatile boolean stop;
    129  /** Field completed */
    130  protected volatile boolean completed = false;
    131
    132  /** Field flowCanonicalHash */
    133  protected String flowCanonicalHash;
    134  /** Field flowElementGraph */
    135  protected FlowElementGraph flowElementGraph; // only used for documentation purposes
    136
    137  private transient CascadingServices cascadingServices;
    138
    139  private FlowStepStrategy<Config> flowStepStrategy = null;
    140  /** Field steps */
    141  protected transient List<FlowStep<Config>> steps;
    142  /** Field jobsMap */
    143  private transient Map<String, FlowStepJob<Config>> jobsMap;
    144  private transient UnitOfWorkSpawnStrategy spawnStrategy = new UnitOfWorkExecutorStrategy();
    145
    146  private transient ReentrantLock stopLock = new ReentrantLock( true );
    147  protected ShutdownUtil.Hook shutdownHook;
    148
    149  protected HashMap<String, String> flowDescriptor;
    150
    151  /**
    152   * Returns property stopJobsOnExit.
    153   *
    154   * @param properties of type Map
    155   * @return a boolean
    156   */
    157  static boolean getStopJobsOnExit( Map<Object, Object> properties )
    158    {
    159    return Boolean.parseBoolean( PropertyUtil.getProperty( properties, FlowProps.STOP_JOBS_ON_EXIT, "true" ) );
    160    }
    161
    162  /** Used for testing. */
    163  protected BaseFlow()
    164    {
    165    this.name = "NA";
    166    this.flowStats = createPrepareFlowStats();
    167    }
    168
    169  /**
    170   * Does not initialize stats
    171   *
    172   * @param name
    173   */
    174  protected BaseFlow( PlatformInfo platformInfo, String name )
    175    {
    176    if( platformInfo != null )
    177      this.platformInfo = platformInfo;
    178
    179    this.name = name;
    180    }
    181
    182  protected BaseFlow( PlatformInfo platformInfo, Map<Object, Object> properties, Config defaultConfig, String name, Map<String, String> flowDescriptor )
    183    {
    184    if( platformInfo != null )
    185      this.platformInfo = platformInfo;
    186
    187    this.name = name;
    188
    189    if( flowDescriptor != null )
    190      this.flowDescriptor = new LinkedHashMap<>( flowDescriptor );
    191
    192    addSessionProperties( properties );
    193    initConfig( properties, defaultConfig );
    194    }
    195
    196  protected BaseFlow( PlatformInfo platformInfo, Map<Object, Object> properties, Config defaultConfig, FlowDef flowDef )
    197    {
    198    properties = PropertyUtil.asFlatMap( properties );
    199
    200    if( platformInfo != null )
    201      this.platformInfo = platformInfo;
    202
    203    this.name = flowDef.getName();
    204    this.tags = flowDef.getTags();
    205    this.runID = flowDef.getRunID();
    206    this.classPath = flowDef.getClassPath();
    207
    208    if( !flowDef.getFlowDescriptor().isEmpty() )
    209      this.flowDescriptor = new LinkedHashMap<>( flowDef.getFlowDescriptor() );
    210
    211    addSessionProperties( properties );
    212    initConfig( properties, defaultConfig );
    213    setSources( flowDef.getSourcesCopy() );
    214    setSinks( flowDef.getSinksCopy() );
    215    setTraps( flowDef.getTrapsCopy() );
    216    setCheckpoints( flowDef.getCheckpointsCopy() );
    217    initFromTaps();
    218
    219    retrieveSourceFields();
    220    retrieveSinkFields();
    221    }
    222
    223  public void setPlannerInfo( PlannerInfo plannerInfo )
    224    {
    225    this.plannerInfo = plannerInfo;
    226    }
    227
    228  @Override
    229  public PlannerInfo getPlannerInfo()
    230    {
    231    return plannerInfo;
    232    }
    233
    234  @Override
    235  public PlatformInfo getPlatformInfo()
    236    {
    237    return platformInfo;
    238    }
    239
    240  public void initialize( FlowElementGraph flowElementGraph, FlowStepGraph flowStepGraph )
    241    {
    242    addPlannerProperties();
    243    this.flowElementGraph = flowElementGraph;
    244    this.flowStepGraph = flowStepGraph;
    245
    246    initSteps();
    247
    248    this.flowStats = createPrepareFlowStats();
    249
    250    initializeNewJobsMap();
    251
    252    initializeChildStats();
    253    }
    254
    255  public FlowElementGraph updateSchemes( FlowElementGraph pipeGraph )
    256    {
    257    presentSourceFields( pipeGraph );
    258
    259    presentSinkFields( pipeGraph );
    260
    261    return new FlowElementGraph( pipeGraph );
    262    }
    263
    264  /** Force a Scheme to fetch any fields from a meta-data store */
    265  protected void retrieveSourceFields()
    266    {
    267    for( Tap tap : sources.values() )
    268      tap.retrieveSourceFields( getFlowProcess() );
    269    }
    270
    271  /**
    272   * Present the current resolved fields for the Tap
    273   *
    274   * @param pipeGraph
    275   */
    276  protected void presentSourceFields( FlowElementGraph pipeGraph )
    277    {
    278    for( Tap tap : sources.values() )
    279      {
    280      if( pipeGraph.containsVertex( tap ) )
    281        tap.presentSourceFields( getFlowProcess(), getFieldsFor( pipeGraph, tap ) );
    282      }
    283
    284    for( Tap tap : checkpoints.values() )
    285      {
    286      if( pipeGraph.containsVertex( tap ) )
    287        tap.presentSourceFields( getFlowProcess(), getFieldsFor( pipeGraph, tap ) );
    288      }
    289    }
    290
    291  /** Force a Scheme to fetch any fields from a meta-data store */
    292  protected void retrieveSinkFields()
    293    {
    294    for( Tap tap : sinks.values() )
    295      tap.retrieveSinkFields( getFlowProcess() );
    296    }
    297
    298  /**
    299   * Present the current resolved fields for the Tap
    300   *
    301   * @param pipeGraph
    302   */
    303  protected void presentSinkFields( FlowElementGraph pipeGraph )
    304    {
    305    for( Tap tap : sinks.values() )
    306      {
    307      if( pipeGraph.containsVertex( tap ) )
    308        tap.presentSinkFields( getFlowProcess(), getFieldsFor( pipeGraph, tap ) );
    309      }
    310
    311    for( Tap tap : checkpoints.values() )
    312      {
    313      if( pipeGraph.containsVertex( tap ) )
    314        tap.presentSinkFields( getFlowProcess(), getFieldsFor( pipeGraph, tap ) );
    315      }
    316    }
    317
    318  protected Fields getFieldsFor( FlowElementGraph pipeGraph, Tap tap )
    319    {
    320    return pipeGraph.outgoingEdgesOf( tap ).iterator().next().getOutValuesFields();
    321    }
    322
    323  protected void addSessionProperties( Map<Object, Object> properties )
    324    {
    325    if( properties == null )
    326      return;
    327
    328    PropertyUtil.setProperty( properties, CASCADING_FLOW_ID, getID() );
    329    PropertyUtil.setProperty( properties, "cascading.flow.tags", getTags() );
    330    AppProps.setApplicationID( properties );
    331    PropertyUtil.setProperty( properties, "cascading.app.name", makeAppName( properties ) );
    332    PropertyUtil.setProperty( properties, "cascading.app.version", makeAppVersion( properties ) );
    333    }
    334
    335  protected void addPlannerProperties()
    336    {
    337    setConfigProperty( getConfig(), "cascading.flow.planner", getPlannerInfo().name );
    338    setConfigProperty( getConfig(), "cascading.flow.platform", getPlannerInfo().platform );
    339    setConfigProperty( getConfig(), "cascading.flow.registry", getPlannerInfo().registry );
    340    }
    341
    342  private String makeAppName( Map<Object, Object> properties )
    343    {
    344    if( properties == null )
    345      return null;
    346
    347    String name = AppProps.getApplicationName( properties );
    348
    349    if( name != null )
    350      return name;
    351
    352    return Util.findName( AppProps.getApplicationJarPath( properties ) );
    353    }
    354
    355  private String makeAppVersion( Map<Object, Object> properties )
    356    {
    357    if( properties == null )
    358      return null;
    359
    360    String name = AppProps.getApplicationVersion( properties );
    361
    362    if( name != null )
    363      return name;
    364
    365    return Util.findVersion( AppProps.getApplicationJarPath( properties ) );
    366    }
    367
    368  protected FlowStats createPrepareFlowStats()
    369    {
    370    FlowStats flowStats = createFlowStats();
    371
    372    flowStats.prepare();
    373    flowStats.markPending();
    374
    375    return flowStats;
    376    }
    377
    378  protected FlowStats createFlowStats()
    379    {
    380    return new FlowStats( this, getClientState() );
    381    }
    382
    383  public CascadingServices getCascadingServices()
    384    {
    385    if( cascadingServices == null )
    386      cascadingServices = new CascadingServices( getConfigAsProperties() );
    387
    388    return cascadingServices;
    389    }
    390
    391  protected ClientState getClientState()
    392    {
    393    return getFlowSession().getCascadingServices().createClientState( getID() );
    394    }
    395
    396  protected void initSteps()
    397    {
    398    if( flowStepGraph == null )
    399      return;
    400
    401    Set<FlowStep> flowSteps = flowStepGraph.vertexSet();
    402
    403    for( FlowStep flowStep : flowSteps )
    404      {
    405      ( (BaseFlowStep) flowStep ).setFlow( this );
    406
    407      Set<FlowNode> flowNodes = flowStep.getFlowNodeGraph().vertexSet();
    408
    409      for( FlowNode flowNode : flowNodes )
    410        ( (BaseFlowNode) flowNode ).setFlowStep( flowStep );
    411      }
    412    }
    413
    414  private void initFromTaps()
    415    {
    416    initFromTaps( sources );
    417    initFromTaps( sinks );
    418    initFromTaps( traps );
    419    }
    420
    421  private void initFromTaps( Map<String, Tap> taps )
    422    {
    423    for( Tap tap : taps.values() )
    424      tap.flowConfInit( this );
    425    }
    426
    427  @Override
    428  public String getName()
    429    {
    430    return name;
    431    }
    432
    433  protected void setName( String name )
    434    {
    435    this.name = name;
    436    }
    437
    438  @Override
    439  public String getID()
    440    {
    441    if( id == null )
    442      id = Util.createUniqueID();
    443
    444    return id;
    445    }
    446
    447  @Override
    448  public String getTags()
    449    {
    450    return tags;
    451    }
    452
    453  @Override
    454  public int getSubmitPriority()
    455    {
    456    return submitPriority;
    457    }
    458
    459  @Override
    460  public void setSubmitPriority( int submitPriority )
    461    {
    462    if( submitPriority < 1 || submitPriority > 10 )
    463      throw new IllegalArgumentException( "submitPriority must be between 1 and 10 inclusive, was: " + submitPriority );
    464
    465    this.submitPriority = submitPriority;
    466    }
    467
    468  /**
    469   * The hash value can be used to determine if two unique Flow instances performed the same work.
    470   * <p/>
    471   * The source and sink taps are not relevant to the hash.
    472   *
    473   * @return a String value
    474   */
    475  public String getFlowCanonicalHash()
    476    {
    477    if( flowCanonicalHash != null || flowElementGraph == null )
    478      return flowCanonicalHash;
    479
    480    // synchronize on flowElementGraph to prevent duplicate hash creation - can be high overhead
    481    // and to prevent deadlocks if the DocumentService calls back into the flow when transitioning
    482    // into a running state
    483    synchronized( flowElementGraph )
    484      {
    485      flowCanonicalHash = createFlowCanonicalHash( flowElementGraph );
    486      }
    487
    488    return flowCanonicalHash;
    489    }
    490
    491  // allow for sub-class overrides
    492  protected String createFlowCanonicalHash( FlowElementGraph flowElementGraph )
    493    {
    494    if( flowElementGraph == null )
    495      return null;
    496
    497    return ElementGraphs.canonicalHash( flowElementGraph );
    498    }
    499
    500  protected FlowElementGraph getFlowElementGraph()
    501    {
    502    return flowElementGraph;
    503    }
    504
    505  protected void setFlowElementGraph( FlowElementGraph flowElementGraph )
    506    {
    507    this.flowElementGraph = flowElementGraph;
    508    }
    509
    510  protected FlowStepGraph getFlowStepGraph()
    511    {
    512    return flowStepGraph;
    513    }
    514
    515  protected void setFlowStepGraph( FlowStepGraph flowStepGraph )
    516    {
    517    this.flowStepGraph = flowStepGraph;
    518    }
    519
    520  protected void setSources( Map<String, Tap> sources )
    521    {
    522    if( sources == null )
    523      return;
    524
    525    addListeners( sources.values() );
    526    this.sources = sources;
    527    }
    528
    529  protected void setSinks( Map<String, Tap> sinks )
    530    {
    531    if( sinks == null )
    532      return;
    533
    534    addListeners( sinks.values() );
    535    this.sinks = sinks;
    536    }
    537
    538  protected void setTraps( Map<String, Tap> traps )
    539    {
    540    addListeners( traps.values() );
    541    this.traps = traps;
    542    }
    543
    544  protected void setCheckpoints( Map<String, Tap> checkpoints )
    545    {
    546    addListeners( checkpoints.values() );
    547    this.checkpoints = checkpoints;
    548    }
    549
    550  /**
    551   * This method creates a new internal Config with the parentConfig as defaults using the properties to override
    552   * the defaults.
    553   *
    554   * @param properties   of type Map
    555   * @param parentConfig of type Config
    556   */
    557  protected abstract void initConfig( Map<Object, Object> properties, Config parentConfig );
    558
    559  public Config createConfig( Map<Object, Object> properties, Config defaultConfig )
    560    {
    561    Config config = newConfig( defaultConfig );
    562
    563    if( properties == null )
    564      return config;
    565
    566    Set<Object> keys = new HashSet<>( properties.keySet() );
    567
    568    // keys will only be grabbed if both key/value are String, so keep orig keys
    569    if( properties instanceof Properties )
    570      keys.addAll( ( (Properties) properties ).stringPropertyNames() );
    571
    572    for( Object key : keys )
    573      {
    574      Object value = properties.get( key );
    575
    576      if( value == null && properties instanceof Properties && key instanceof String )
    577        value = ( (Properties) properties ).getProperty( (String) key );
    578
    579      if( value == null ) // don't stuff null values
    580        continue;
    581
    582      setConfigProperty( config, key, value );
    583      }
    584
    585    return config;
    586    }
    587
    588  protected abstract void setConfigProperty( Config config, Object key, Object value );
    589
    590  protected abstract Config newConfig( Config defaultConfig );
    591
    592  protected void initFromProperties( Map<Object, Object> properties )
    593    {
    594    stopJobsOnExit = getStopJobsOnExit( properties );
    595    }
    596
    597  public FlowSession getFlowSession()
    598    {
    599    return new FlowSession( getCascadingServices() );
    600    }
    601
    602  @Override
    603  public FlowStats getFlowStats()
    604    {
    605    return flowStats;
    606    }
    607
    608  @Override
    609  public Map<String, String> getFlowDescriptor()
    610    {
    611    if( flowDescriptor == null )
    612      return Collections.emptyMap();
    613
    614    return Collections.unmodifiableMap( flowDescriptor );
    615    }
    616
    617  @Override
    618  public FlowStats getStats()
    619    {
    620    return getFlowStats();
    621    }
    622
    623  void addListeners( Collection listeners )
    624    {
    625    for( Object listener : listeners )
    626      {
    627      if( listener instanceof FlowListener )
    628        addListener( (FlowListener) listener );
    629      }
    630    }
    631
    632  protected void removeListeners( Collection listeners )
    633    {
    634    if( listeners == null || listeners.isEmpty() )
    635      return;
    636
    637    for( Object listener : listeners )
    638      {
    639      if( listener instanceof FlowListener )
    640        removeListener( (FlowListener) listener );
    641      }
    642    }
    643
    644  List<SafeFlowListener> getListeners()
    645    {
    646    if( listeners == null )
    647      listeners = new LinkedList<SafeFlowListener>();
    648
    649    return listeners;
    650    }
    651
    652  @Override
    653  public boolean hasListeners()
    654    {
    655    return listeners != null && !listeners.isEmpty();
    656    }
    657
    658  @Override
    659  public void addListener( FlowListener flowListener )
    660    {
    661    getListeners().add( new SafeFlowListener( flowListener ) );
    662    }
    663
    664  @Override
    665  public boolean removeListener( FlowListener flowListener )
    666    {
    667    return getListeners().remove( new SafeFlowListener( flowListener ) );
    668    }
    669
    670  @Override
    671  public boolean hasStepListeners()
    672    {
    673    boolean hasStepListeners = false;
    674
    675    for( FlowStep step : getFlowSteps() )
    676      hasStepListeners |= step.hasListeners();
    677
    678    return hasStepListeners;
    679    }
    680
    681  @Override
    682  public void addStepListener( FlowStepListener flowStepListener )
    683    {
    684    for( FlowStep step : getFlowSteps() )
    685      step.addListener( flowStepListener );
    686    }
    687
    688  @Override
    689  public boolean removeStepListener( FlowStepListener flowStepListener )
    690    {
    691    boolean listenerRemoved = true;
    692
    693    for( FlowStep step : getFlowSteps() )
    694      listenerRemoved &= step.removeListener( flowStepListener );
    695
    696    return listenerRemoved;
    697    }
    698
    699  @Override
    700  public Map<String, Tap> getSources()
    701    {
    702    return Collections.unmodifiableMap( sources );
    703    }
    704
    705  @Override
    706  public List<String> getSourceNames()
    707    {
    708    return new ArrayList<String>( sources.keySet() );
    709    }
    710
    711  @Override
    712  public Tap getSource( String name )
    713    {
    714    return sources.get( name );
    715    }
    716
    717  @Override
    718  @DependencyIncoming
    719  public Collection<Tap> getSourcesCollection()
    720    {
    721    return getSources().values();
    722    }
    723
    724  @Override
    725  public Map<String, Tap> getSinks()
    726    {
    727    return Collections.unmodifiableMap( sinks );
    728    }
    729
    730  @Override
    731  public List<String> getSinkNames()
    732    {
    733    return new ArrayList<String>( sinks.keySet() );
    734    }
    735
    736  @Override
    737  public Tap getSink( String name )
    738    {
    739    return sinks.get( name );
    740    }
    741
    742  @Override
    743  @DependencyOutgoing
    744  public Collection<Tap> getSinksCollection()
    745    {
    746    return getSinks().values();
    747    }
    748
    749  @Override
    750  public Tap getSink()
    751    {
    752    return sinks.values().iterator().next();
    753    }
    754
    755  @Override
    756  public Map<String, Tap> getTraps()
    757    {
    758    return Collections.unmodifiableMap( traps );
    759    }
    760
    761  @Override
    762  public List<String> getTrapNames()
    763    {
    764    return new ArrayList<String>( traps.keySet() );
    765    }
    766
    767  @Override
    768  public Collection<Tap> getTrapsCollection()
    769    {
    770    return getTraps().values();
    771    }
    772
    773  @Override
    774  public Map<String, Tap> getCheckpoints()
    775    {
    776    return Collections.unmodifiableMap( checkpoints );
    777    }
    778
    779  @Override
    780  public List<String> getCheckpointNames()
    781    {
    782    return new ArrayList<String>( checkpoints.keySet() );
    783    }
    784
    785  @Override
    786  public Collection<Tap> getCheckpointsCollection()
    787    {
    788    return getCheckpoints().values();
    789    }
    790
    791  @Override
    792  public boolean isStopJobsOnExit()
    793    {
    794    return stopJobsOnExit;
    795    }
    796
    797  @Override
    798  public FlowSkipStrategy getFlowSkipStrategy()
    799    {
    800    return flowSkipStrategy;
    801    }
    802
    803  @Override
    804  public FlowSkipStrategy setFlowSkipStrategy( FlowSkipStrategy flowSkipStrategy )
    805    {
    806    if( flowSkipStrategy == null )
    807      throw new IllegalArgumentException( "flowSkipStrategy may not be null" );
    808
    809    try
    810      {
    811      return this.flowSkipStrategy;
    812      }
    813    finally
    814      {
    815      this.flowSkipStrategy = flowSkipStrategy;
    816      }
    817    }
    818
    819  @Override
    820  public boolean isSkipFlow() throws IOException
    821    {
    822    return flowSkipStrategy.skipFlow( this );
    823    }
    824
    825  @Override
    826  public boolean areSinksStale() throws IOException
    827    {
    828    return areSourcesNewer( getSinkModified() );
    829    }
    830
    831  @Override
    832  public boolean areSourcesNewer( long sinkModified ) throws IOException
    833    {
    834    Config config = getConfig();
    835    Iterator<Tap> values = sources.values().iterator();
    836
    837    long sourceModified = 0;
    838
    839    try
    840      {
    841      sourceModified = Util.getSourceModified( config, values, sinkModified );
    842
    843      if( sinkModified < sourceModified )
    844        return true;
    845
    846      return false;
    847      }
    848    finally
    849      {
    850      if( LOG.isInfoEnabled() )
    851        logInfo( "source modification date at: " + new Date( sourceModified ) ); // not oldest, we didnt check them all
    852      }
    853    }
    854
    855  @Override
    856  public long getSinkModified() throws IOException
    857    {
    858    long sinkModified = Util.getSinkModified( getConfig(), sinks.values() );
    859
    860    if( LOG.isInfoEnabled() )
    861      {
    862      if( sinkModified == -1L )
    863        logInfo( "at least one sink is marked for delete" );
    864      if( sinkModified == 0L )
    865        logInfo( "at least one sink does not exist" );
    866      else
    867        logInfo( "sink oldest modified date: " + new Date( sinkModified ) );
    868      }
    869
    870    return sinkModified;
    871    }
    872
    873  @Override
    874  public FlowStepStrategy getFlowStepStrategy()
    875    {
    876    return flowStepStrategy;
    877    }
    878
    879  @Override
    880  public void setFlowStepStrategy( FlowStepStrategy flowStepStrategy )
    881    {
    882    this.flowStepStrategy = flowStepStrategy;
    883    }
    884
    885  @Override
    886  public List<FlowStep<Config>> getFlowSteps()
    887    {
    888    if( steps != null )
    889      return steps;
    890
    891    if( flowStepGraph == null )
    892      return Collections.emptyList();
    893
    894    Iterator<FlowStep> topoIterator = flowStepGraph.getTopologicalIterator();
    895
    896    steps = new ArrayList<>();
    897
    898    while( topoIterator.hasNext() )
    899      steps.add( topoIterator.next() );
    900
    901    return steps;
    902    }
    903
    904  @Override
    905  @ProcessPrepare
    906  public void prepare()
    907    {
    908    try
    909      {
    910      deleteSinksIfNotUpdate();
    911      deleteTrapsIfNotUpdate();
    912      deleteCheckpointsIfNotUpdate();
    913      }
    914    catch( IOException exception )
    915      {
    916      throw new FlowException( "unable to prepare flow", exception );
    917      }
    918    }
    919
    920  @Override
    921  @ProcessStart
    922  public synchronized void start()
    923    {
    924    if( thread != null )
    925      return;
    926
    927    if( stop )
    928      return;
    929
    930    registerShutdownHook();
    931
    932    internalStart();
    933
    934    String threadName = ( "flow " + Util.toNull( getName() ) ).trim();
    935
    936    thread = createFlowThread( threadName );
    937
    938    thread.start();
    939    }
    940
    941  protected Thread createFlowThread( String threadName )
    942    {
    943    return new Thread( new Runnable()
    944      {
    945      @Override
    946      public void run()
    947        {
    948        BaseFlow.this.run();
    949        }
    950      }, threadName );
    951    }
    952
    953  protected abstract void internalStart();
    954
    955  @Override
    956  @ProcessStop
    957  public synchronized void stop()
    958    {
    959    stopLock.lock();
    960
    961    try
    962      {
    963      if( stop )
    964        return;
    965
    966      stop = true;
    967
    968      fireOnStopping();
    969
    970      if( !flowStats.isFinished() )
    971        flowStats.markStopped();
    972
    973      internalStopAllJobs();
    974
    975      handleExecutorShutdown();
    976
    977      internalClean( true );
    978      }
    979    finally
    980      {
    981      flowStats.cleanup();
    982      stopLock.unlock();
    983      }
    984    }
    985
    986  protected abstract void internalClean( boolean stop );
    987
    988  @Override
    989  @ProcessComplete
    990  public void complete()
    991    {
    992    if( !completed )
    993      start();
    994
    995    try
    996      {
    997      try
    998        {
    999        synchronized( this ) // prevent NPE on quick stop() & complete() after start()
    1000          {
    1001          while( !completed && thread == null && !stop )
    1002            Util.safeSleep( 10 );
    1003          }
    1004
    1005        if( thread != null )
    1006          thread.join();
    1007        }
    1008      catch( InterruptedException exception )
    1009        {
    1010        throw new FlowException( getName(), "thread interrupted", exception );
    1011        }
    1012
    1013      // if in #stop and stopping, lets wait till its done in this thread
    1014      try
    1015        {
    1016        stopLock.lock();
    1017        }
    1018      finally
    1019        {
    1020        stopLock.unlock();
    1021        }
    1022
    1023      // if multiple threads enter #complete, each will get a copy of the exception, if any
    1024      if( throwable instanceof FlowException )
    1025        ( (FlowException) throwable ).setFlowName( getName() );
    1026
    1027      if( throwable instanceof CascadingException )
    1028        throw (CascadingException) throwable;
    1029
    1030      if( throwable instanceof OutOfMemoryError )
    1031        throw (OutOfMemoryError) throwable;
    1032
    1033      if( throwable != null )
    1034        throw new FlowException( getName(), "unhandled exception", throwable );
    1035
    1036      if( hasListeners() )
    1037        {
    1038        for( SafeFlowListener safeFlowListener : getListeners() )
    1039          {
    1040          if( safeFlowListener.throwable != null )
    1041            throw new FlowException( getName(), "unhandled listener exception", throwable );
    1042          }
    1043        }
    1044      }
    1045    finally
    1046      {
    1047      completed = true;
    1048      thread = null;
    1049      throwable = null;
    1050      }
    1051    }
    1052
    1053  private void commitTraps()
    1054    {
    1055    // commit all the traps, don't fail on an error
    1056    for( Tap tap : traps.values() )
    1057      {
    1058      try
    1059        {
    1060        if( !tap.commitResource( getConfig() ) )
    1061          logError( "unable to commit trap: " + tap.getFullIdentifier( getConfig() ) );
    1062        }
    1063      catch( IOException exception )
    1064        {
    1065        logError( "unable to commit trap: " + tap.getFullIdentifier( getConfig() ), exception );
    1066        }
    1067      }
    1068    }
    1069
    1070  @Override
    1071  @ProcessCleanup
    1072  public void cleanup()
    1073    {
    1074    // do nothing
    1075    }
    1076
    1077  @Override
    1078  public TupleEntryIterator openSource() throws IOException
    1079    {
    1080    return sources.values().iterator().next().openForRead( getFlowProcess() );
    1081    }
    1082
    1083  @Override
    1084  public TupleEntryIterator openSource( String name ) throws IOException
    1085    {
    1086    if( !sources.containsKey( name ) )
    1087      throw new IllegalArgumentException( "source does not exist: " + name );
    1088
    1089    return sources.get( name ).openForRead( getFlowProcess() );
    1090    }
    1091
    1092  @Override
    1093  public TupleEntryIterator openSink() throws IOException
    1094    {
    1095    return sinks.values().iterator().next().openForRead( getFlowProcess() );
    1096    }
    1097
    1098  @Override
    1099  public TupleEntryIterator openSink( String name ) throws IOException
    1100    {
    1101    if( !sinks.containsKey( name ) )
    1102      throw new IllegalArgumentException( "sink does not exist: " + name );
    1103
    1104    return sinks.get( name ).openForRead( getFlowProcess() );
    1105    }
    1106
    1107  @Override
    1108  public TupleEntryIterator openTrap() throws IOException
    1109    {
    1110    return traps.values().iterator().next().openForRead( getFlowProcess() );
    1111    }
    1112
    1113  @Override
    1114  public TupleEntryIterator openTrap( String name ) throws IOException
    1115    {
    1116    if( !traps.containsKey( name ) )
    1117      throw new IllegalArgumentException( "trap does not exist: " + name );
    1118
    1119    return traps.get( name ).openForRead( getFlowProcess() );
    1120    }
    1121
    1122  /**
    1123   * Method deleteSinks deletes all sinks, whether or not they are configured for {@link cascading.tap.SinkMode#UPDATE}.
    1124   * <p/>
    1125   * Use with caution.
    1126   *
    1127   * @throws IOException when
    1128   * @see BaseFlow#deleteSinksIfNotUpdate()
    1129   */
    1130  public void deleteSinks() throws IOException
    1131    {
    1132    for( Tap tap : sinks.values() )
    1133      deleteOrFail( tap );
    1134    }
    1135
    1136  private void deleteOrFail( Tap tap ) throws IOException
    1137    {
    1138    if( !tap.resourceExists( getConfig() ) )
    1139      return;
    1140
    1141    if( !tap.deleteResource( getConfig() ) )
    1142      throw new FlowException( "unable to delete resource: " + tap.getFullIdentifier( getFlowProcess() ) );
    1143    }
    1144
    1145  /**
    1146   * Method deleteSinksIfNotUpdate deletes all sinks if they are not configured with the {@link cascading.tap.SinkMode#UPDATE} flag.
    1147   * <p/>
    1148   * Typically used by a {@link Cascade} before executing the flow if the sinks are stale.
    1149   * <p/>
    1150   * Use with caution.
    1151   *
    1152   * @throws IOException when
    1153   */
    1154  public void deleteSinksIfNotUpdate() throws IOException
    1155    {
    1156    for( Tap tap : sinks.values() )
    1157      {
    1158      if( !tap.isUpdate() )
    1159        deleteOrFail( tap );
    1160      }
    1161    }
    1162
    1163  public void deleteSinksIfReplace() throws IOException
    1164    {
    1165    for( Tap tap : sinks.values() )
    1166      {
    1167      if( tap.isReplace() )
    1168        deleteOrFail( tap );
    1169      }
    1170    }
    1171
    1172  public void deleteTrapsIfNotUpdate() throws IOException
    1173    {
    1174    for( Tap tap : traps.values() )
    1175      {
    1176      if( !tap.isUpdate() )
    1177        deleteOrFail( tap );
    1178      }
    1179    }
    1180
    1181  public void deleteCheckpointsIfNotUpdate() throws IOException
    1182    {
    1183    for( Tap tap : checkpoints.values() )
    1184      {
    1185      if( !tap.isUpdate() )
    1186        deleteOrFail( tap );
    1187      }
    1188    }
    1189
    1190  public void deleteTrapsIfReplace() throws IOException
    1191    {
    1192    for( Tap tap : traps.values() )
    1193      {
    1194      if( tap.isReplace() )
    1195        deleteOrFail( tap );
    1196      }
    1197    }
    1198
    1199  public void deleteCheckpointsIfReplace() throws IOException
    1200    {
    1201    for( Tap tap : checkpoints.values() )
    1202      {
    1203      if( tap.isReplace() )
    1204        deleteOrFail( tap );
    1205      }
    1206    }
    1207
    1208  @Override
    1209  public boolean resourceExists( Tap tap ) throws IOException
    1210    {
    1211    return tap.resourceExists( getConfig() );
    1212    }
    1213
    1214  @Override
    1215  public TupleEntryIterator openTapForRead( Tap tap ) throws IOException
    1216    {
    1217    return tap.openForRead( getFlowProcess() );
    1218    }
    1219
    1220  @Override
    1221  public TupleEntryCollector openTapForWrite( Tap tap ) throws IOException
    1222    {
    1223    return tap.openForWrite( getFlowProcess() );
    1224    }
    1225
    1226  /** Method run implements the Runnable run method and should not be called by users. */
    1227  private void run()
    1228    {
    1229    if( thread == null )
    1230      throw new IllegalStateException( "to start a Flow call start() or complete(), not Runnable#run()" );
    1231
    1232    Version.printBanner();
    1233    Update.checkForUpdate( getPlatformInfo() );
    1234
    1235    try
    1236      {
    1237      if( stop )
    1238        return;
    1239
    1240      flowStats.markStarted();
    1241
    1242      fireOnStarting();
    1243
    1244      if( isInfoEnabled() )
    1245        {
    1246        logInfo( "starting" );
    1247
    1248        for( Tap source : getSourcesCollection() )
    1249          logInfo( " source: " + source );
    1250        for( Tap sink : getSinksCollection() )
    1251          logInfo( " sink: " + sink );
    1252        }
    1253
    1254      spawnSteps();
    1255      }
    1256    catch( Throwable throwable )
    1257      {
    1258      this.throwable = throwable;
    1259      }
    1260    finally
    1261      {
    1262      handleThrowableAndMarkFailed();
    1263
    1264      if( !stop && !flowStats.isFinished() )
    1265        flowStats.markSuccessful();
    1266
    1267      internalClean( stop ); // cleaning temp taps may be determined by success/failure
    1268
    1269      commitTraps();
    1270
    1271      try
    1272        {
    1273        fireOnCompleted();
    1274        }
    1275      finally
    1276        {
    1277        if( LOG.isInfoEnabled() )
    1278          {
    1279          long totalSliceCPUSeconds = getTotalSliceCPUMilliSeconds();
    1280
    1281          if( totalSliceCPUSeconds == -1 )
    1282            logInfo( " completed in: " + formatDurationFromMillis( flowStats.getDuration() ) );
    1283          else
    1284            logInfo( " completed in: " + formatDurationFromMillis( flowStats.getDuration() ) + ", using cpu time: " + formatDurationFromMillis( totalSliceCPUSeconds ) );
    1285          }
    1286
    1287        flowStats.cleanup();
    1288        internalShutdown();
    1289        deregisterShutdownHook();
    1290        }
    1291      }
    1292    }
    1293
    1294  /**
    1295   * Reentrant method to launch all step jobs, returns false if any of the current steps failed.
    1296   *
    1297   * @return
    1298   * @throws InterruptedException
    1299   * @throws ExecutionException
    1300   */
    1301  protected boolean spawnSteps() throws InterruptedException, ExecutionException
    1302    {
    1303    // if jobs are run local, then only use one thread to force execution serially
    1304    //int numThreads = jobsAreLocal() ? 1 : getMaxConcurrentSteps( getJobConf() );
    1305    int numThreads = getMaxNumParallelSteps();
    1306    int eligibleJobsSize = getEligibleJobsSize(); // only jobs that have not previously been started
    1307
    1308    if( numThreads == 0 )
    1309      numThreads = eligibleJobsSize;
    1310
    1311    if( numThreads == 0 )
    1312      throw new IllegalStateException( "no jobs rendered for flow: " + getName() );
    1313
    1314    if( LOG.isInfoEnabled() )
    1315      {
    1316      logInfo( " parallel execution of steps is enabled: " + ( getMaxNumParallelSteps() != 1 ) );
    1317      logInfo( " executing total steps: " + eligibleJobsSize );
    1318      logInfo( " allocating management threads: " + numThreads );
    1319      }
    1320
    1321    List<Future<Throwable>> futures = spawnJobs( numThreads );
    1322
    1323    for( Future<Throwable> future : futures )
    1324      {
    1325      throwable = future.get();
    1326
    1327      if( throwable != null )
    1328        {
    1329        if( !stop )
    1330          internalStopAllJobs();
    1331
    1332        handleExecutorShutdown();
    1333        return false;
    1334        }
    1335      }
    1336
    1337    return true;
    1338    }
    1339
    1340  protected long getTotalSliceCPUMilliSeconds()
    1341    {
    1342    return -1;
    1343    }
    1344
    1345  protected abstract int getMaxNumParallelSteps();
    1346
    1347  protected abstract void internalShutdown();
    1348
    1349  private List<Future<Throwable>> spawnJobs( int numThreads ) throws InterruptedException
    1350    {
    1351    if( spawnStrategy == null )
    1352      {
    1353      logError( "no spawnStrategy set" );
    1354      return new ArrayList<>();
    1355      }
    1356
    1357    if( stop )
    1358      return new ArrayList<>();
    1359
    1360    List<Callable<Throwable>> list = getJobMapCallables();
    1361
    1362    return spawnStrategy.start( this, numThreads, list );
    1363    }
    1364
    1365  private void handleThrowableAndMarkFailed()
    1366    {
    1367    if( throwable != null && !stop )
    1368      {
    1369      flowStats.markFailed( throwable );
    1370
    1371      fireOnThrowable();
    1372      }
    1373    }
    1374
    1375  Map<String, FlowStepJob<Config>> getJobsMap()
    1376    {
    1377    return jobsMap;
    1378    }
    1379
    1380  protected boolean isJobsMapInitialized()
    1381    {
    1382    return jobsMap != null;
    1383    }
    1384
    1385  protected int getEligibleJobsSize()
    1386    {
    1387    return getJobMapCallables().size();
    1388    }
    1389
    1390  protected List<Callable<Throwable>> getJobMapCallables()
    1391    {
    1392    List<Callable<Throwable>> list = new ArrayList<>();
    1393
    1394    for( FlowStepJob<Config> job : jobsMap.values() )
    1395      {
    1396      if( job.isCallableStarted() )
    1397        continue;
    1398
    1399      list.add( job );
    1400      }
    1401
    1402    return list;
    1403    }
    1404
    1405  protected void initializeNewJobsMap()
    1406    {
    1407    jobsMap = updateJobsMap( flowStepGraph, new LinkedHashMap<String, FlowStepJob<Config>>() ); // keep topo order
    1408    }
    1409
    1410  protected void updateJobsMap()
    1411    {
    1412    jobsMap = updateJobsMap( flowStepGraph, new LinkedHashMap<>( jobsMap ) );
    1413    }
    1414
    1415  Map<String, FlowStepJob<Config>> updateJobsMap( FlowStepGraph flowStepGraph, Map<String, FlowStepJob<Config>> jobsMap )
    1416    {
    1417    Iterator<FlowStep> iterator = flowStepGraph.getTopologicalIterator();
    1418
    1419    while( iterator.hasNext() )
    1420      {
    1421      BaseFlowStep<Config> step = (BaseFlowStep) iterator.next();
    1422      FlowStepJob<Config> flowStepJob = jobsMap.get( step.getID() );
    1423
    1424      if( flowStepJob == null )
    1425        {
    1426        flowStepJob = step.getCreateFlowStepJob( getFlowProcess(), getConfig() );
    1427
    1428        jobsMap.put( step.getID(), flowStepJob );
    1429        }
    1430
    1431      List<FlowStepJob<Config>> predecessors = new ArrayList<>();
    1432
    1433      for( Object flowStep : ProcessGraphs.predecessorListOf( flowStepGraph, step ) )
    1434        predecessors.add( jobsMap.get( ( (FlowStep) flowStep ).getID() ) );
    1435
    1436      flowStepJob.setPredecessors( predecessors );
    1437      }
    1438
    1439    return jobsMap;
    1440    }
    1441
    1442  protected void initializeChildStats()
    1443    {
    1444    for( FlowStepJob<Config> flowStepJob : jobsMap.values() )
    1445      flowStats.addStepStats( flowStepJob.getStepStats() );
    1446    }
    1447
    1448  protected void internalStopAllJobs()
    1449    {
    1450    logInfo( "stopping all jobs" );
    1451
    1452    try
    1453      {
    1454      if( jobsMap == null )
    1455        return;
    1456
    1457      List<FlowStepJob<Config>> jobs = new ArrayList<FlowStepJob<Config>>( jobsMap.values() );
    1458
    1459      Collections.reverse( jobs );
    1460
    1461      for( FlowStepJob<Config> job : jobs )
    1462        job.stop();
    1463      }
    1464    finally
    1465      {
    1466      logInfo( "stopped all jobs" );
    1467      }
    1468    }
    1469
    1470  protected void handleExecutorShutdown()
    1471    {
    1472    if( spawnStrategy == null )
    1473      return;
    1474
    1475    if( spawnStrategy.isCompleted( this ) )
    1476      return;
    1477
    1478    logDebug( "shutting down job executor" );
    1479
    1480    try
    1481      {
    1482      spawnStrategy.complete( this, 5 * 60, TimeUnit.SECONDS );
    1483      }
    1484    catch( InterruptedException exception )
    1485      {
    1486      // ignore
    1487      }
    1488
    1489    logDebug( "shutdown of job executor complete" );
    1490    }
    1491
    1492  protected void fireOnCompleted()
    1493    {
    1494    if( hasListeners() )
    1495      {
    1496      if( isDebugEnabled() )
    1497        logDebug( "firing onCompleted event: " + getListeners().size() );
    1498
    1499      for( FlowListener flowListener : getListeners() )
    1500        flowListener.onCompleted( this );
    1501      }
    1502    }
    1503
    1504  protected void fireOnThrowable( Throwable throwable )
    1505    {
    1506    this.throwable = throwable;
    1507    fireOnThrowable();
    1508    }
    1509
    1510  protected void fireOnThrowable()
    1511    {
    1512    if( hasListeners() )
    1513      {
    1514      if( isDebugEnabled() )
    1515        logDebug( "firing onThrowable event: " + getListeners().size() );
    1516
    1517      boolean isHandled = false;
    1518
    1519      for( FlowListener flowListener : getListeners() )
    1520        isHandled = flowListener.onThrowable( this, throwable ) || isHandled;
    1521
    1522      if( isHandled )
    1523        throwable = null;
    1524      }
    1525    }
    1526
    1527  protected void fireOnStopping()
    1528    {
    1529    if( hasListeners() )
    1530      {
    1531      if( isDebugEnabled() )
    1532        logDebug( "firing onStopping event: " + getListeners().size() );
    1533
    1534      for( FlowListener flowListener : getListeners() )
    1535        flowListener.onStopping( this );
    1536      }
    1537    }
    1538
    1539  protected void fireOnStarting()
    1540    {
    1541    if( hasListeners() )
    1542      {
    1543      if( isDebugEnabled() )
    1544        logDebug( "firing onStarting event: " + getListeners().size() );
    1545
    1546      for( FlowListener flowListener : getListeners() )
    1547        flowListener.onStarting( this );
    1548      }
    1549    }
    1550
    1551  @Override
    1552  public String toString()
    1553    {
    1554    StringBuffer buffer = new StringBuffer();
    1555
    1556    if( getName() != null )
    1557      buffer.append( getName() ).append( ": " );
    1558
    1559    for( FlowStep step : getFlowSteps() )
    1560      buffer.append( step );
    1561
    1562    return buffer.toString();
    1563    }
    1564
    1565  @Override
    1566  public final boolean isInfoEnabled()
    1567    {
    1568    return LOG.isInfoEnabled();
    1569    }
    1570
    1571  @Override
    1572  public final boolean isDebugEnabled()
    1573    {
    1574    return LOG.isDebugEnabled();
    1575    }
    1576
    1577  @Override
    1578  public void logInfo( String message, Object... arguments )
    1579    {
    1580    LOG.info( "[" + Util.truncate( getName(), LOG_FLOW_NAME_MAX ) + "] " + message, arguments );
    1581    }
    1582
    1583  @Override
    1584  public void logDebug( String message, Object... arguments )
    1585    {
    1586    LOG.debug( "[" + Util.truncate( getName(), LOG_FLOW_NAME_MAX ) + "] " + message, arguments );
    1587    }
    1588
    1589  @Override
    1590  public void logWarn( String message )
    1591    {
    1592    LOG.warn( "[" + Util.truncate( getName(), LOG_FLOW_NAME_MAX ) + "] " + message );
    1593    }
    1594
    1595  @Override
    1596  public void logWarn( String message, Throwable throwable )
    1597    {
    1598    LOG.warn( "[" + Util.truncate( getName(), LOG_FLOW_NAME_MAX ) + "] " + message, throwable );
    1599    }
    1600
    1601  @Override
    1602  public void logWarn( String message, Object... arguments )
    1603    {
    1604    LOG.warn( "[" + Util.truncate( getName(), LOG_FLOW_NAME_MAX ) + "] " + message, arguments );
    1605    }
    1606
    1607  @Override
    1608  public void logError( String message, Object... arguments )
    1609    {
    1610    LOG.error( "[" + Util.truncate( getName(), LOG_FLOW_NAME_MAX ) + "] " + message, arguments );
    1611    }
    1612
    1613  @Override
    1614  public void logError( String message, Throwable throwable )
    1615    {
    1616    LOG.error( "[" + Util.truncate( getName(), LOG_FLOW_NAME_MAX ) + "] " + message, throwable );
    1617    }
    1618
    1619  @Override
    1620  public void writeDOT( String filename )
    1621    {
    1622    if( flowElementGraph == null )
    1623      throw new UnsupportedOperationException( "this flow instance cannot write a DOT file" );
    1624
    1625    flowElementGraph.writeDOT( filename );
    1626    }
    1627
    1628  @Override
    1629  public void writeStepsDOT( String filename )
    1630    {
    1631    if( flowStepGraph == null )
    1632      throw new UnsupportedOperationException( "this flow instance cannot write a DOT file" );
    1633
    1634    flowStepGraph.writeDOT( filename );
    1635    }
    1636
    1637  /**
    1638   * Used to return a simple wrapper for use as an edge in a graph where there can only be
    1639   * one instance of every edge.
    1640   *
    1641   * @return FlowHolder
    1642   */
    1643  public FlowHolder getHolder()
    1644    {
    1645    return new FlowHolder( this );
    1646    }
    1647
    1648  public void setCascade( Cascade cascade )
    1649    {
    1650    setConfigProperty( getConfig(), "cascading.cascade.id", cascade.getID() );
    1651    flowStats.recordInfo();
    1652    }
    1653
    1654  @Override
    1655  public String getCascadeID()
    1656    {
    1657    return getProperty( "cascading.cascade.id" );
    1658    }
    1659
    1660  @Override
    1661  public String getRunID()
    1662    {
    1663    return runID;
    1664    }
    1665
    1666  public List<String> getClassPath()
    1667    {
    1668    return classPath;
    1669    }
    1670
    1671  @Override
    1672  public void setSpawnStrategy( UnitOfWorkSpawnStrategy spawnStrategy )
    1673    {
    1674    this.spawnStrategy = spawnStrategy;
    1675    }
    1676
    1677  @Override
    1678  public UnitOfWorkSpawnStrategy getSpawnStrategy()
    1679    {
    1680    return spawnStrategy;
    1681    }
    1682
    1683  protected void registerShutdownHook()
    1684    {
    1685    if( !isStopJobsOnExit() )
    1686      return;
    1687
    1688    shutdownHook = new ShutdownUtil.Hook()
    1689      {
    1690      @Override
    1691      public Priority priority()
    1692        {
    1693        return Priority.WORK_CHILD;
    1694        }
    1695
    1696      @Override
    1697      public void execute()
    1698        {
    1699        logInfo( "shutdown hook calling stop on flow" );
    1700
    1701        BaseFlow.this.stop();
    1702        }
    1703      };
    1704
    1705    ShutdownUtil.addHook( shutdownHook );
    1706    }
    1707
    1708  private void deregisterShutdownHook()
    1709    {
    1710    if( !isStopJobsOnExit() || stop )
    1711      return;
    1712
    1713    ShutdownUtil.removeHook( shutdownHook );
    1714    }
    1715
    1716  /** Class FlowHolder is a helper class for wrapping Flow instances. */
    1717  public static class FlowHolder
    1718    {
    1719    /** Field flow */
    1720    public Flow flow;
    1721
    1722    public FlowHolder()
    1723      {
    1724      }
    1725
    1726    public FlowHolder( Flow flow )
    1727      {
    1728      this.flow = flow;
    1729      }
    1730    }
    1731
    1732  /**
    1733   * Class SafeFlowListener safely calls a wrapped FlowListener.
    1734   * <p/>
    1735   * This is done for a few reasons, the primary reason is so exceptions thrown by the Listener
    1736   * can be caught by the calling Thread. Since Flow is asynchronous, much of the work is done in the run() method
    1737   * which in turn is run in a new Thread.
    1738   */
    1739  private class SafeFlowListener implements FlowListener
    1740    {
    1741    /** Field flowListener */
    1742    final FlowListener flowListener;
    1743    /** Field throwable */
    1744    Throwable throwable;
    1745
    1746    private SafeFlowListener( FlowListener flowListener )
    1747      {
    1748      this.flowListener = flowListener;
    1749      }
    1750
    1751    public void onStarting( Flow flow )
    1752      {
    1753      try
    1754        {
    1755        flowListener.onStarting( flow );
    1756        }
    1757      catch( Throwable throwable )
    1758        {
    1759        handleThrowable( throwable );
    1760        }
    1761      }
    1762
    1763    public void onStopping( Flow flow )
    1764      {
    1765      try
    1766        {
    1767        flowListener.onStopping( flow );
    1768        }
    1769      catch( Throwable throwable )
    1770        {
    1771        handleThrowable( throwable );
    1772        }
    1773      }
    1774
    1775    public void onCompleted( Flow flow )
    1776      {
    1777      try
    1778        {
    1779        flowListener.onCompleted( flow );
    1780        }
    1781      catch( Throwable throwable )
    1782        {
    1783        handleThrowable( throwable );
    1784        }
    1785      }
    1786
    1787    public boolean onThrowable( Flow flow, Throwable flowThrowable )
    1788      {
    1789      try
    1790        {
    1791        return flowListener.onThrowable( flow, flowThrowable );
    1792        }
    1793      catch( Throwable throwable )
    1794        {
    1795        handleThrowable( throwable );
    1796        }
    1797
    1798      return false;
    1799      }
    1800
    1801    private void handleThrowable( Throwable throwable )
    1802      {
    1803      this.throwable = throwable;
    1804
    1805      logWarn( String.format( "flow listener %s threw throwable", flowListener ), throwable );
    1806
    1807      // stop this flow
    1808      stop();
    1809      }
    1810
    1811    public boolean equals( Object object )
    1812      {
    1813      if( object instanceof BaseFlow.SafeFlowListener )
    1814        return flowListener.equals( ( (BaseFlow.SafeFlowListener) object ).flowListener );
    1815
    1816      return flowListener.equals( object );
    1817      }
    1818
    1819    public int hashCode()
    1820      {
    1821      return flowListener.hashCode();
    1822      }
    1823    }
    1824  }