cascading.flow.planner
Class BaseFlowStep<Config>

java.lang.Object
  extended by cascading.flow.planner.BaseFlowStep<Config>
All Implemented Interfaces:
FlowStep<Config>, java.io.Serializable
Direct Known Subclasses:
HadoopFlowStep, LocalFlowStep

public abstract class BaseFlowStep<Config>
extends java.lang.Object
implements java.io.Serializable, FlowStep<Config>

Class FlowStep is an internal representation of a given Job to be executed on a remote cluster. During planning, pipe assemblies are broken down into "steps" and encapsulated in this class.

FlowSteps are submitted in order of dependency. If two or more steps do not share the same dependencies and all can be scheduled simultaneously, the getSubmitPriority() value determines the order in which all steps will be submitted for execution. The default submit priority is 5.

This class is for internal use, there are no stable public methods.

See Also:
Serialized Form

Field Summary
protected  java.util.Map<HashJoin,java.util.Set<Tap>> accumulatedSourcesByJoin
           
protected  java.util.Map<Tap,java.util.Set<java.lang.String>> sinks
          Field sink
protected  java.util.Map<Tap,java.util.Set<java.lang.String>> sources
          Field sources
protected  java.util.Map<HashJoin,Tap> streamedSourceByJoin
           
protected  Tap tempSink
          Field tempSink
 
Fields inherited from interface cascading.flow.FlowStep
CASCADING_FLOW_STEP_ID
 
Constructor Summary
protected BaseFlowStep(java.lang.String name, int stepNum)
           
 
Method Summary
 void addAccumulatedSourceFor(HashJoin join, Tap accumulatedSource)
           
 void addGroup(Group group)
           
 void addSink(java.lang.String name, Tap sink)
           
 void addSource(java.lang.String name, Tap source)
           
 void addStreamedSourceFor(HashJoin join, Tap streamedSource)
           
abstract  void clean(Config config)
           
 void commitSinks()
           
 boolean containsPipeNamed(java.lang.String pipeName)
          Returns true if this FlowStep contains a pipe/branch with the given name.
protected  ClientState createClientState(FlowProcess flowProcess)
           
protected abstract  FlowStepJob createFlowStepJob(FlowProcess<Config> flowProcess, Config parentConfig)
           
 boolean equals(java.lang.Object object)
           
 java.util.Set<Tap> getAllAccumulatedSources()
           
 java.util.Collection<Operation> getAllOperations()
           
 Config getConfig()
          Method getConfig returns the current initialized configuration.
 Flow<Config> getFlow()
           
 java.lang.String getFlowID()
           
 java.lang.String getFlowName()
          Method getParentFlowName returns the parentFlowName of this FlowStep object.
 FlowStepJob<Config> getFlowStepJob(FlowProcess<Config> flowProcess, Config parentConfig)
           
 FlowStepStats getFlowStepStats()
           
 org.jgrapht.graph.SimpleDirectedGraph<FlowElement,Scope> getGraph()
           
 Group getGroup()
           
 java.util.List<Group> getGroups()
           
 java.lang.String getID()
          Method getId returns the id of this FlowStep object.
protected abstract  Config getInitializedConfig(FlowProcess<Config> flowProcess, Config parentConfig)
           
 java.util.Set<Tap> getJoinTributariesBetween(FlowElement from, FlowElement to)
           
 java.lang.String getName()
          Method getName returns the name of this FlowStep object.
 FlowElement getNextFlowElement(Scope scope)
           
 Scope getNextScope(FlowElement flowElement)
          Method getNextScope returns the next Scope instance in the graph.
 java.util.Set<Scope> getNextScopes(FlowElement flowElement)
           
 java.util.Set<Scope> getPreviousScopes(FlowElement flowElement)
          Method getPreviousScopes returns the previous Scope instances.
 Scope getScopeFor(FlowElement sourceElement, FlowElement targetElement)
           
 Tap getSink()
           
 java.util.Set<java.lang.String> getSinkName(Tap sink)
           
 java.util.Set<Tap> getSinks()
           
 Tap getSinkWith(java.lang.String identifier)
           
 java.util.Set<java.lang.String> getSourceName(Tap source)
           
 java.util.Set<Tap> getSources()
           
 Tap getSourceWith(java.lang.String identifier)
           
 java.lang.String getStepDisplayName()
          Method getStepDisplayName returns the stepDisplayName of this FlowStep object.
 int getStepNum()
           
 java.util.Map<HashJoin,Tap> getStreamedSourceByJoin()
           
 int getSubmitPriority()
          Method getSubmitPriority returns the submitPriority of this FlowStep object.
 java.util.List<FlowElement> getSuccessors(FlowElement element)
           
 org.jgrapht.traverse.TopologicalOrderIterator<FlowElement,Scope> getTopologicalOrderIterator()
           
 int hashCode()
           
protected  void initConfFromProcessConfigDef(ConfigDef.Setter setter)
           
 boolean isDebugEnabled()
           
 boolean isInfoEnabled()
           
 void logDebug(java.lang.String message)
           
 void logError(java.lang.String message, java.lang.Throwable throwable)
           
 void logInfo(java.lang.String message)
           
 void logWarn(java.lang.String message)
           
 void logWarn(java.lang.String message, java.lang.Throwable throwable)
           
 void rollbackSinks()
           
protected  void setConf(Config conf)
           
 void setFlow(Flow<Config> flow)
           
protected  void setFlowName(java.lang.String flowName)
           
 void setSubmitPriority(int submitPriority)
          Method setSubmitPriority sets the submitPriority of this FlowStep object.
 java.lang.String toString()
           
 
Methods inherited from class java.lang.Object
clone, finalize, getClass, notify, notifyAll, wait, wait, wait
 
Methods inherited from interface cascading.flow.FlowStep
getTrap, getTraps
 

Field Detail

sources

protected final java.util.Map<Tap,java.util.Set<java.lang.String>> sources
Field sources


sinks

protected final java.util.Map<Tap,java.util.Set<java.lang.String>> sinks
Field sink


tempSink

protected Tap tempSink
Field tempSink


streamedSourceByJoin

protected final java.util.Map<HashJoin,Tap> streamedSourceByJoin

accumulatedSourcesByJoin

protected final java.util.Map<HashJoin,java.util.Set<Tap>> accumulatedSourcesByJoin
Constructor Detail

BaseFlowStep

protected BaseFlowStep(java.lang.String name,
                       int stepNum)
Method Detail

getID

public java.lang.String getID()
Description copied from interface: FlowStep
Method getId returns the id of this FlowStep object.

Specified by:
getID in interface FlowStep<Config>
Returns:
the id (type int) of this FlowStep object.

getStepNum

public int getStepNum()
Specified by:
getStepNum in interface FlowStep<Config>

getName

public java.lang.String getName()
Description copied from interface: FlowStep
Method getName returns the name of this FlowStep object.

Specified by:
getName in interface FlowStep<Config>
Returns:
the name (type String) of this FlowStep object.

setFlow

public void setFlow(Flow<Config> flow)

getFlow

public Flow<Config> getFlow()
Specified by:
getFlow in interface FlowStep<Config>

getFlowID

public java.lang.String getFlowID()
Specified by:
getFlowID in interface FlowStep<Config>

getFlowName

public java.lang.String getFlowName()
Description copied from interface: FlowStep
Method getParentFlowName returns the parentFlowName of this FlowStep object.

Specified by:
getFlowName in interface FlowStep<Config>
Returns:
the parentFlowName (type Flow) of this FlowStep object.

setFlowName

protected void setFlowName(java.lang.String flowName)

getConfig

public Config getConfig()
Description copied from interface: FlowStep
Method getConfig returns the current initialized configuration.

The returned configuration is mutable and may be changed prior to this step being started or submitted.

Specified by:
getConfig in interface FlowStep<Config>
Returns:
the current initialized configuration

setConf

protected void setConf(Config conf)

getStepDisplayName

public java.lang.String getStepDisplayName()
Description copied from interface: FlowStep
Method getStepDisplayName returns the stepDisplayName of this FlowStep object.

Specified by:
getStepDisplayName in interface FlowStep<Config>
Returns:
the stepName (type String) of this FlowStep object.

getSubmitPriority

public int getSubmitPriority()
Description copied from interface: FlowStep
Method getSubmitPriority returns the submitPriority of this FlowStep object.

10 is lowest, 1 is the highest, 5 is the default.

Specified by:
getSubmitPriority in interface FlowStep<Config>
Returns:
the submitPriority (type int) of this FlowStep object.

setSubmitPriority

public void setSubmitPriority(int submitPriority)
Description copied from interface: FlowStep
Method setSubmitPriority sets the submitPriority of this FlowStep object.

10 is lowest, 1 is the highest, 5 is the default.

Specified by:
setSubmitPriority in interface FlowStep<Config>
Parameters:
submitPriority - the submitPriority of this FlowStep object.

getFlowStepStats

public FlowStepStats getFlowStepStats()

getGraph

public org.jgrapht.graph.SimpleDirectedGraph<FlowElement,Scope> getGraph()

getGroup

public Group getGroup()
Specified by:
getGroup in interface FlowStep<Config>

getGroups

public java.util.List<Group> getGroups()
Specified by:
getGroups in interface FlowStep<Config>

addGroup

public void addGroup(Group group)

getStreamedSourceByJoin

public java.util.Map<HashJoin,Tap> getStreamedSourceByJoin()
Specified by:
getStreamedSourceByJoin in interface FlowStep<Config>

addStreamedSourceFor

public void addStreamedSourceFor(HashJoin join,
                                 Tap streamedSource)

getAllAccumulatedSources

public java.util.Set<Tap> getAllAccumulatedSources()
Specified by:
getAllAccumulatedSources in interface FlowStep<Config>

addAccumulatedSourceFor

public void addAccumulatedSourceFor(HashJoin join,
                                    Tap accumulatedSource)

addSource

public void addSource(java.lang.String name,
                      Tap source)

addSink

public void addSink(java.lang.String name,
                    Tap sink)

getSources

public java.util.Set<Tap> getSources()
Specified by:
getSources in interface FlowStep<Config>

getSinks

public java.util.Set<Tap> getSinks()
Specified by:
getSinks in interface FlowStep<Config>

getSink

public Tap getSink()
Specified by:
getSink in interface FlowStep<Config>

getSourceName

public java.util.Set<java.lang.String> getSourceName(Tap source)
Specified by:
getSourceName in interface FlowStep<Config>

getSinkName

public java.util.Set<java.lang.String> getSinkName(Tap sink)
Specified by:
getSinkName in interface FlowStep<Config>

getSourceWith

public Tap getSourceWith(java.lang.String identifier)
Specified by:
getSourceWith in interface FlowStep<Config>

getSinkWith

public Tap getSinkWith(java.lang.String identifier)
Specified by:
getSinkWith in interface FlowStep<Config>

commitSinks

public void commitSinks()

rollbackSinks

public void rollbackSinks()

getInitializedConfig

protected abstract Config getInitializedConfig(FlowProcess<Config> flowProcess,
                                               Config parentConfig)

getPreviousScopes

public java.util.Set<Scope> getPreviousScopes(FlowElement flowElement)
Method getPreviousScopes returns the previous Scope instances. If the flowElement is a Group (specifically a CoGroup), there will be more than one instance.

Parameters:
flowElement - of type FlowElement
Returns:
Set

getNextScope

public Scope getNextScope(FlowElement flowElement)
Method getNextScope returns the next Scope instance in the graph. There will always only be one next.

Parameters:
flowElement - of type FlowElement
Returns:
Scope

getScopeFor

public Scope getScopeFor(FlowElement sourceElement,
                         FlowElement targetElement)

getNextScopes

public java.util.Set<Scope> getNextScopes(FlowElement flowElement)

getNextFlowElement

public FlowElement getNextFlowElement(Scope scope)

getTopologicalOrderIterator

public org.jgrapht.traverse.TopologicalOrderIterator<FlowElement,Scope> getTopologicalOrderIterator()

getSuccessors

public java.util.List<FlowElement> getSuccessors(FlowElement element)

getJoinTributariesBetween

public java.util.Set<Tap> getJoinTributariesBetween(FlowElement from,
                                                    FlowElement to)

getAllOperations

public java.util.Collection<Operation> getAllOperations()

containsPipeNamed

public boolean containsPipeNamed(java.lang.String pipeName)
Description copied from interface: FlowStep
Returns true if this FlowStep contains a pipe/branch with the given name.

Specified by:
containsPipeNamed in interface FlowStep<Config>
Returns:

clean

public abstract void clean(Config config)

equals

public boolean equals(java.lang.Object object)
Overrides:
equals in class java.lang.Object

createClientState

protected ClientState createClientState(FlowProcess flowProcess)

getFlowStepJob

public FlowStepJob<Config> getFlowStepJob(FlowProcess<Config> flowProcess,
                                          Config parentConfig)

createFlowStepJob

protected abstract FlowStepJob createFlowStepJob(FlowProcess<Config> flowProcess,
                                                 Config parentConfig)

initConfFromProcessConfigDef

protected void initConfFromProcessConfigDef(ConfigDef.Setter setter)

hashCode

public int hashCode()
Overrides:
hashCode in class java.lang.Object

toString

public java.lang.String toString()
Overrides:
toString in class java.lang.Object

isInfoEnabled

public final boolean isInfoEnabled()

isDebugEnabled

public final boolean isDebugEnabled()

logDebug

public void logDebug(java.lang.String message)

logInfo

public void logInfo(java.lang.String message)

logWarn

public void logWarn(java.lang.String message)

logWarn

public void logWarn(java.lang.String message,
                    java.lang.Throwable throwable)

logError

public void logError(java.lang.String message,
                     java.lang.Throwable throwable)


Copyright © 2007-2011 Concurrent, Inc. All Rights Reserved.