    001/*
    002 * Copyright (c) 2016-2017 Chris K Wensel <chris@wensel.net>. All Rights Reserved.
    003 * Copyright (c) 2007-2017 Xplenty, Inc. All Rights Reserved.
    004 *
    005 * Project and contact information: http://www.cascading.org/
    006 *
    007 * This file is part of the Cascading project.
    008 *
    009 * Licensed under the Apache License, Version 2.0 (the "License");
    010 * you may not use this file except in compliance with the License.
    011 * You may obtain a copy of the License at
    012 *
    013 *     http://www.apache.org/licenses/LICENSE-2.0
    014 *
    015 * Unless required by applicable law or agreed to in writing, software
    016 * distributed under the License is distributed on an "AS IS" BASIS,
    017 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    018 * See the License for the specific language governing permissions and
    019 * limitations under the License.
    020 */
    021
    022package cascading.flow.local.planner;
    023
    024import java.util.ArrayList;
    025import java.util.Collection;
    026import java.util.Collections;
    027import java.util.List;
    028import java.util.Properties;
    029import java.util.concurrent.Callable;
    030import java.util.concurrent.CancellationException;
    031import java.util.concurrent.ExecutorService;
    032import java.util.concurrent.Executors;
    033import java.util.concurrent.Future;
    034import java.util.concurrent.Semaphore;
    035
    036import cascading.flow.FlowNode;
    037import cascading.flow.FlowProcess;
    038import cascading.flow.local.LocalFlowStep;
    039import cascading.flow.local.stream.graph.LocalStepStreamGraph;
    040import cascading.flow.stream.duct.Duct;
    041import cascading.flow.stream.graph.StreamGraph;
    042import cascading.util.Util;
    043import org.slf4j.Logger;
    044import org.slf4j.LoggerFactory;
    045
    046import static cascading.util.LogUtil.logCounters;
    047import static cascading.util.LogUtil.logMemory;
    048
    049/**
    050 *
    051 */
    052public class LocalStepRunner implements Callable<Throwable>
    053  {
    054  private static final Logger LOG = LoggerFactory.getLogger( LocalStepRunner.class );
    055
    056  private final FlowProcess<Properties> currentProcess;
    057
    058  private volatile boolean started = false;
    059  private volatile boolean completed = false;
    060  private volatile boolean successful = false;
    061  private volatile boolean stopped = false;
    062
    063  private final FlowNode flowNode;
    064  private final StreamGraph streamGraph;
    065  private final Collection<Duct> heads;
    066  private Throwable throwable = null;
    067
    068  private Semaphore markComplete = new Semaphore( 0 );
    069  private List<Future<Throwable>> futures = Collections.emptyList();
    070
    071  public LocalStepRunner( FlowProcess<Properties> flowProcess, LocalFlowStep step )
    072    {
    073    this.currentProcess = flowProcess;
    074    this.flowNode = Util.getFirst( step.getFlowNodeGraph().vertexSet() );
    075    this.streamGraph = new LocalStepStreamGraph( this.currentProcess, step, flowNode );
    076    this.heads = streamGraph.getHeads();
    077    }
    078
    079  public FlowProcess<Properties> getFlowProcess()
    080    {
    081    return currentProcess;
    082    }
    083
    084  public boolean isCompleted()
    085    {
    086    return completed;
    087    }
    088
    089  public void blockUntilStopped()
    090    {
    091    if( !started || completed )
    092      return;
    093
    094    stopped = true;
    095
    096    for( Future<Throwable> future : futures )
    097      future.cancel( true );
    098
    099    try
    100      {
    101      markComplete.acquire();
    102      }
    103    catch( InterruptedException exception )
    104      {
    105      // do nothing
    106      }
    107    }
    108
    109  public boolean isStopped()
    110    {
    111    return stopped;
    112    }
    113
    114  public boolean isSuccessful()
    115    {
    116    return successful;
    117    }
    118
    119  public Throwable getThrowable()
    120    {
    121    return throwable;
    122    }
    123
    124  @Override
    125  public Throwable call() throws Exception
    126    {
    127    started = true;
    128    boolean attemptedCleanup = false;
    129
    130    try
    131      {
    132      try
    133        {
    134        streamGraph.prepare();
    135
    136        logMemory( LOG, "flow node id: " + flowNode.getID() + ", mem on start" );
    137        }
    138      catch( Throwable currentThrowable )
    139        {
    140        try
    141          {
    142          if( !( currentThrowable instanceof OutOfMemoryError ) )
    143            LOG.error( "unable to prepare operation graph", currentThrowable );
    144
    145          completed = true;
    146          successful = false;
    147          throwable = currentThrowable;
    148
    149          return throwable;
    150          }
    151        finally
    152          {
    153          markComplete.release();
    154          }
    155        }
    156
    157      if( stopped )
    158        {
    159        markComplete.release();
    160
    161        return null;
    162        }
    163
    164      try
    165        {
    166        futures = spawnHeads();
    167
    168        for( Future<Throwable> future : futures )
    169          {
    170          try
    171            {
    172            throwable = future.get();
    173            }
    174          catch( CancellationException exception )
    175            {
    176            break;
    177            }
    178
    179          if( throwable != null )
    180            break;
    181          }
    182
    183        futures = Collections.emptyList();
    184        }
    185      catch( Throwable currentThrowable )
    186        {
    187        if( !( currentThrowable instanceof OutOfMemoryError ) )
    188          LOG.error( "unable to complete step", currentThrowable );
    189
    190        throwable = currentThrowable;
    191        }
    192
    193      try
    194        {
    195        attemptedCleanup = true; // set so we don't try again regardless
    196
    197        if( !( throwable instanceof OutOfMemoryError ) )
    198          streamGraph.cleanup();
    199        }
    200      catch( Throwable currentThrowable )
    201        {
    202        if( !( currentThrowable instanceof OutOfMemoryError ) )
    203          LOG.error( "unable to cleanup operation graph", currentThrowable );
    204
    205        if( throwable == null )
    206          throwable = currentThrowable;
    207        }
    208
    209      completed = true;
    210      successful = throwable == null;
    211
    212      return throwable;
    213      }
    214    finally
    215      {
    216      try
    217        {
    218        if( !attemptedCleanup )
    219          streamGraph.cleanup();
    220        }
    221      catch( Throwable currentThrowable )
    222        {
    223        if( !( currentThrowable instanceof OutOfMemoryError ) )
    224          LOG.error( "unable to cleanup operation graph", currentThrowable );
    225
    226        if( throwable == null )
    227          throwable = currentThrowable;
    228
    229        successful = false;
    230        }
    231      finally
    232        {
    233        markComplete.release();
    234        }
    235
    236      String message = "flow node id: " + flowNode.getID();
    237      logMemory( LOG, message + ", mem on close" );
    238      logCounters( LOG, message + ", counter:", currentProcess );
    239      }
    240    }
    241
    242  private List<Future<Throwable>> spawnHeads()
    243    {
    244    // todo: consider a CyclicBarrier to syn all threads after the openForRead
    245    // todo: should find all Callable Ducts and spawn them, group ducts may run on a timer etc
    246    ExecutorService executors = Executors.newFixedThreadPool( heads.size() );
    247    List<Future<Throwable>> futures = new ArrayList<Future<Throwable>>();
    248
    249    for( Duct head : heads )
    250      futures.add( executors.submit( (Callable) head ) );
    251
    252    executors.shutdown();
    253
    254    return futures;
    255    }
    256  }