    001/*
    002 * Copyright (c) 2016 Chris K Wensel <chris@wensel.net>. All Rights Reserved.
    003 * Copyright (c) 2007-2017 Xplenty, Inc. All Rights Reserved.
    004 *
    005 * Project and contact information: http://www.cascading.org/
    006 *
    007 * This file is part of the Cascading project.
    008 *
    009 * Licensed under the Apache License, Version 2.0 (the "License");
    010 * you may not use this file except in compliance with the License.
    011 * You may obtain a copy of the License at
    012 *
    013 *     http://www.apache.org/licenses/LICENSE-2.0
    014 *
    015 * Unless required by applicable law or agreed to in writing, software
    016 * distributed under the License is distributed on an "AS IS" BASIS,
    017 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    018 * See the License for the specific language governing permissions and
    019 * limitations under the License.
    020 */
    021
    022package cascading.flow.tez;
    023
    024import java.io.File;
    025import java.io.IOException;
    026import java.util.ArrayList;
    027import java.util.Collection;
    028import java.util.HashMap;
    029import java.util.HashSet;
    030import java.util.Iterator;
    031import java.util.LinkedList;
    032import java.util.List;
    033import java.util.Map;
    034import java.util.Set;
    035
    036import cascading.CascadingException;
    037import cascading.flow.FlowElement;
    038import cascading.flow.FlowElements;
    039import cascading.flow.FlowException;
    040import cascading.flow.FlowNode;
    041import cascading.flow.FlowProcess;
    042import cascading.flow.FlowRuntimeProps;
    043import cascading.flow.hadoop.ConfigurationSetter;
    044import cascading.flow.hadoop.util.HadoopUtil;
    045import cascading.flow.planner.BaseFlowStep;
    046import cascading.flow.planner.FlowStepJob;
    047import cascading.flow.planner.graph.ElementGraph;
    048import cascading.flow.planner.process.FlowNodeGraph;
    049import cascading.flow.planner.process.ProcessEdge;
    050import cascading.flow.stream.annotations.StreamMode;
    051import cascading.flow.tez.planner.Hadoop2TezFlowStepJob;
    052import cascading.flow.tez.util.TezUtil;
    053import cascading.management.state.ClientState;
    054import cascading.pipe.Boundary;
    055import cascading.pipe.CoGroup;
    056import cascading.pipe.Group;
    057import cascading.pipe.GroupBy;
    058import cascading.pipe.Merge;
    059import cascading.pipe.Splice;
    060import cascading.property.AppProps;
    061import cascading.tap.Tap;
    062import cascading.tap.hadoop.Hfs;
    063import cascading.tap.hadoop.PartitionTap;
    064import cascading.tap.hadoop.util.Hadoop18TapUtil;
    065import cascading.tuple.Fields;
    066import cascading.tuple.hadoop.TupleSerialization;
    067import cascading.tuple.hadoop.util.GroupingSortingComparator;
    068import cascading.tuple.hadoop.util.ReverseGroupingSortingComparator;
    069import cascading.tuple.hadoop.util.ReverseTupleComparator;
    070import cascading.tuple.hadoop.util.TupleComparator;
    071import cascading.tuple.io.KeyTuple;
    072import cascading.tuple.io.TuplePair;
    073import cascading.tuple.io.ValueTuple;
    074import cascading.tuple.tez.util.GroupingSortingPartitioner;
    075import cascading.tuple.tez.util.TuplePartitioner;
    076import cascading.util.Util;
    077import cascading.util.Version;
    078import org.apache.hadoop.conf.Configuration;
    079import org.apache.hadoop.fs.FileSystem;
    080import org.apache.hadoop.fs.Path;
    081import org.apache.hadoop.mapred.JobConf;
    082import org.apache.hadoop.mapreduce.JobContext;
    083import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat;
    084import org.apache.hadoop.yarn.api.records.LocalResource;
    085import org.apache.hadoop.yarn.api.records.LocalResourceType;
    086import org.apache.tez.common.TezUtils;
    087import org.apache.tez.dag.api.DAG;
    088import org.apache.tez.dag.api.DataSinkDescriptor;
    089import org.apache.tez.dag.api.DataSourceDescriptor;
    090import org.apache.tez.dag.api.Edge;
    091import org.apache.tez.dag.api.EdgeProperty;
    092import org.apache.tez.dag.api.GroupInputEdge;
    093import org.apache.tez.dag.api.InputDescriptor;
    094import org.apache.tez.dag.api.OutputDescriptor;
    095import org.apache.tez.dag.api.ProcessorDescriptor;
    096import org.apache.tez.dag.api.TezConfiguration;
    097import org.apache.tez.dag.api.UserPayload;
    098import org.apache.tez.dag.api.Vertex;
    099import org.apache.tez.dag.api.VertexGroup;
    100import org.apache.tez.mapreduce.input.MRInput;
    101import org.apache.tez.mapreduce.output.MROutput;
    102import org.apache.tez.runtime.library.api.TezRuntimeConfiguration;
    103import org.apache.tez.runtime.library.input.ConcatenatedMergedKeyValueInput;
    104import org.apache.tez.runtime.library.input.OrderedGroupedKVInput;
    105import org.apache.tez.runtime.library.input.OrderedGroupedMergedKVInput;
    106import org.apache.tez.runtime.library.input.UnorderedKVInput;
    107import org.apache.tez.runtime.library.output.OrderedPartitionedKVOutput;
    108import org.apache.tez.runtime.library.output.UnorderedKVOutput;
    109import org.apache.tez.runtime.library.output.UnorderedPartitionedKVOutput;
    110import org.slf4j.Logger;
    111import org.slf4j.LoggerFactory;
    112
    113import static cascading.flow.hadoop.util.HadoopUtil.*;
    114import static cascading.flow.tez.util.TezUtil.addToClassPath;
    115import static cascading.tap.hadoop.DistCacheTap.CASCADING_LOCAL_RESOURCES;
    116import static cascading.tap.hadoop.DistCacheTap.CASCADING_REMOTE_RESOURCES;
    117import static cascading.util.Util.getFirst;
    118import static java.util.Collections.singletonList;
    119import static org.apache.hadoop.yarn.api.records.LocalResourceType.ARCHIVE;
    120import static org.apache.hadoop.yarn.api.records.LocalResourceType.FILE;
    121
    122/**
    123 *
    124 */
    125public class Hadoop2TezFlowStep extends BaseFlowStep<TezConfiguration>
    126  {
    127  private static final Logger LOG = LoggerFactory.getLogger( Hadoop2TezFlowStep.class );
    128
    129  private Map<String, LocalResource> allLocalResources = new HashMap<>();
    130  private Map<Path, Path> syncPaths = new HashMap<>();
    131  private Map<String, String> environment = new HashMap<>();
    132
    133  public Hadoop2TezFlowStep( ElementGraph elementGraph, FlowNodeGraph flowNodeGraph )
    134    {
    135    super( elementGraph, flowNodeGraph );
    136    }
    137
    138  @Override
    139  public Map<Object, Object> getConfigAsProperties()
    140    {
    141    return HadoopUtil.createProperties( getConfig() );
    142    }
    143
    144  @Override
    145  public TezConfiguration createInitializedConfig( FlowProcess<TezConfiguration> flowProcess, TezConfiguration parentConfig )
    146    {
    147    TezConfiguration stepConf = parentConfig == null ? new TezConfiguration() : new TezConfiguration( parentConfig );
    148
    149    TupleSerialization.setSerializations( stepConf );
    150
    151    String versionString = Version.getRelease();
    152
    153    if( versionString != null )
    154      stepConf.set( "cascading.version", versionString );
    155
    156    stepConf.set( CASCADING_FLOW_STEP_ID, getID() );
    157    stepConf.set( "cascading.flow.step.num", Integer.toString( getOrdinal() ) );
    158
    159    String flowStagingPath = ( (Hadoop2TezFlow) getFlow() ).getFlowStagingPath();
    160    List<String> classPath = ( (Hadoop2TezFlow) getFlow() ).getClassPath();
    161
    162    // is updated in addToClassPath method
    163    Map<String, LocalResource> dagResources = new HashMap<>();
    164
    165    if( !classPath.isEmpty() )
    166      {
    167      // jars in the root will be in the remote CLASSPATH, no need to add to the environment
    168      Map<Path, Path> dagClassPath = addToClassPath( stepConf, flowStagingPath, null, classPath, FILE, dagResources, null );
    169
    170      syncPaths.putAll( dagClassPath );
    171      }
    172
    173    String appJarPath = stepConf.get( AppProps.APP_JAR_PATH );
    174
    175    if( appJarPath != null )
    176      {
    177      // the PATTERN represents the insides of the app jar, those elements must be added to the remote CLASSPATH
    178      List<String> classpath = singletonList( appJarPath );
    179      Map<Path, Path> pathMap = addToClassPath( stepConf, flowStagingPath, null, classpath, ARCHIVE, dagResources, environment );
    180
    181      syncPaths.putAll( pathMap );
    182
    183      // AM does not support environments like containers do, so the classpath has to be passed via configuration.
    184      String fileName = new File( appJarPath ).getName();
    185      stepConf.set( TezConfiguration.TEZ_CLUSTER_ADDITIONAL_CLASSPATH_PREFIX,
    186        "$PWD/" + fileName + "/:$PWD/" + fileName + "/classes/:$PWD/" + fileName + "/lib/*:" );
    187      }
    188
    189    allLocalResources.putAll( dagResources );
    190
    191    initFromStepConfigDef( stepConf );
    192
    193    return stepConf;
    194    }
    195
    196  @Override
    197  protected FlowStepJob createFlowStepJob( ClientState clientState, FlowProcess<TezConfiguration> flowProcess, TezConfiguration initializedStepConfig )
    198    {
    199    DAG dag = createDAG( flowProcess, initializedStepConfig );
    200
    201    return new Hadoop2TezFlowStepJob( clientState, this, initializedStepConfig, dag );
    202    }
    203
    204  private DAG createDAG( FlowProcess<TezConfiguration> flowProcess, TezConfiguration initializedConfig )
    205    {
    206    FlowNodeGraph nodeGraph = getFlowNodeGraph();
    207    Map<FlowNode, Vertex> vertexMap = new HashMap<>();
    208    DAG dag = DAG.create( getStepDisplayName( initializedConfig.getInt( "cascading.display.id.truncate", Util.ID_LENGTH ) ) );
    209
    210    dag.addTaskLocalFiles( allLocalResources );
    211
    212    Iterator<FlowNode> iterator = nodeGraph.getOrderedTopologicalIterator(); // ordering of nodes for consistent remote debugging
    213
    214    while( iterator.hasNext() )
    215      {
    216      FlowNode flowNode = iterator.next();
    217
    218      Vertex vertex = createVertex( flowProcess, initializedConfig, flowNode );
    219      dag.addVertex( vertex );
    220
    221      vertexMap.put( flowNode, vertex );
    222      }
    223
    224    LinkedList<ProcessEdge> processedEdges = new LinkedList<>();
    225
    226    for( ProcessEdge processEdge : nodeGraph.edgeSet() )
    227      {
    228      if( processedEdges.contains( processEdge ) )
    229        continue;
    230
    231      FlowNode edgeTargetFlowNode = nodeGraph.getEdgeTarget( processEdge );
    232
    233      FlowElement flowElement = processEdge.getFlowElement();
    234      List<FlowNode> sourceNodes = nodeGraph.getElementSourceProcesses( flowElement );
    235
    236      EdgeProperty edgeProperty = createEdgeProperty( initializedConfig, processEdge );
    237
    238      Vertex targetVertex = vertexMap.get( edgeTargetFlowNode );
    239
    240      if( sourceNodes.size() == 1 || flowElement instanceof CoGroup || flowElement instanceof Boundary ) // todo: create group vertices around incoming ordinal
    241        {
    242        FlowNode edgeSourceFlowNode = nodeGraph.getEdgeSource( processEdge );
    243        Vertex sourceVertex = vertexMap.get( edgeSourceFlowNode );
    244
    245        LOG.debug( "adding edge between: {} and {}", sourceVertex, targetVertex );
    246
    247        dag.addEdge( Edge.create( sourceVertex, targetVertex, edgeProperty ) );
    248        }
    249      else if( flowElement instanceof GroupBy || flowElement instanceof Merge ) // merge - source nodes > 1
    250        {
    251        List<String> sourceVerticesIDs = new ArrayList<>();
    252        List<Vertex> sourceVertices = new ArrayList<>();
    253
    254        for( FlowNode edgeSourceFlowNode : sourceNodes )
    255          {
    256          sourceVerticesIDs.add( edgeSourceFlowNode.getID() );
    257          sourceVertices.add( vertexMap.get( edgeSourceFlowNode ) );
    258          processedEdges.add( nodeGraph.getEdge( edgeSourceFlowNode, edgeTargetFlowNode ) );
    259          }
    260
    261        VertexGroup vertexGroup = dag.createVertexGroup( edgeTargetFlowNode.getID(), sourceVertices.toArray( new Vertex[ sourceVertices.size() ] ) );
    262
    263        String inputClassName = flowElement instanceof Group ? OrderedGroupedMergedKVInput.class.getName() : ConcatenatedMergedKeyValueInput.class.getName();
    264
    265        InputDescriptor inputDescriptor = InputDescriptor.create( inputClassName ).setUserPayload( edgeProperty.getEdgeDestination().getUserPayload() );
    266
    267        String type = ( (Splice) flowElement ).isMerge() ? "merged" : "grouped";
    268        LOG.info( "adding {} edge between: {} and {}", type, Util.join( sourceVerticesIDs, "," ), targetVertex.getName() );
    269        dag.addEdge( GroupInputEdge.create( vertexGroup, targetVertex, edgeProperty, inputDescriptor ) );
    270        }
    271      else
    272        {
    273        throw new UnsupportedOperationException( "can't make edge for: " + flowElement );
    274        }
    275      }
    276
    277    return dag;
    278    }
    279
    280  private EdgeProperty createEdgeProperty( TezConfiguration config, ProcessEdge processEdge )
    281    {
    282    FlowElement flowElement = processEdge.getFlowElement();
    283
    284    EdgeValues edgeValues = new EdgeValues( new TezConfiguration( config ), processEdge );
    285
    286    edgeValues.keyClassName = KeyTuple.class.getName(); // TEZ_RUNTIME_INTERMEDIATE_OUTPUT_KEY_CLASS
    287    edgeValues.valueClassName = ValueTuple.class.getName(); // TEZ_RUNTIME_INTERMEDIATE_OUTPUT_VALUE_CLASS
    288    edgeValues.keyComparatorClassName = TupleComparator.class.getName();
    289    edgeValues.keyPartitionerClassName = TuplePartitioner.class.getName();
    290    edgeValues.outputClassName = null;
    291    edgeValues.inputClassName = null;
    292    edgeValues.movementType = null;
    293    edgeValues.sourceType = null;
    294    edgeValues.schedulingType = null;
    295
    296    if( flowElement instanceof Group )
    297      applyGroup( edgeValues );
    298    else if( ( flowElement instanceof Boundary || flowElement instanceof Merge ) && processEdge.getSinkAnnotations().contains( StreamMode.Accumulated ) )
    299      applyBoundaryMergeAccumulated( edgeValues );
    300    else if( flowElement instanceof Boundary || flowElement instanceof Merge )
    301      applyBoundaryMerge( edgeValues );
    302    else
    303      throw new IllegalStateException( "unsupported flow element: " + flowElement.getClass().getCanonicalName() );
    304
    305    applyEdgeAnnotations( processEdge, edgeValues );
    306
    307    return createEdgeProperty( edgeValues );
    308    }
    309
    310  private void applyEdgeAnnotations( ProcessEdge processEdge, EdgeValues edgeValues )
    311    {
    312    processEdge.addEdgeAnnotation( edgeValues.movementType );
    313    processEdge.addEdgeAnnotation( edgeValues.sourceType );
    314    processEdge.addEdgeAnnotation( edgeValues.schedulingType );
    315    }
    316
    317  private EdgeValues applyBoundaryMerge( EdgeValues edgeValues )
    318    {
    319    // todo: support for one to one
    320    edgeValues.outputClassName = UnorderedPartitionedKVOutput.class.getName();
    321    edgeValues.inputClassName = UnorderedKVInput.class.getName();
    322
    323    edgeValues.movementType = EdgeProperty.DataMovementType.SCATTER_GATHER;
    324    edgeValues.sourceType = EdgeProperty.DataSourceType.PERSISTED;
    325    edgeValues.schedulingType = EdgeProperty.SchedulingType.SEQUENTIAL;
    326
    327    return edgeValues;
    328    }
    329
    330  private EdgeValues applyBoundaryMergeAccumulated( EdgeValues edgeValues )
    331    {
    332    edgeValues.outputClassName = UnorderedKVOutput.class.getName();
    333    edgeValues.inputClassName = UnorderedKVInput.class.getName();
    334
    335    edgeValues.movementType = EdgeProperty.DataMovementType.BROADCAST;
    336    edgeValues.sourceType = EdgeProperty.DataSourceType.PERSISTED;
    337    edgeValues.schedulingType = EdgeProperty.SchedulingType.SEQUENTIAL;
    338
    339    return edgeValues;
    340    }
    341
    342  private EdgeValues applyGroup( EdgeValues edgeValues )
    343    {
    344    Group group = (Group) edgeValues.flowElement;
    345
    346    if( group.isSortReversed() )
    347      edgeValues.keyComparatorClassName = ReverseTupleComparator.class.getName();
    348
    349    int ordinal = getFirst( edgeValues.ordinals );
    350
    351    addComparators( edgeValues.config, "cascading.group.comparator", group.getKeySelectors(), edgeValues.getResolvedKeyFieldsMap().get( ordinal ) );
    352
    353    if( !group.isGroupBy() )
    354      {
    355      edgeValues.outputClassName = OrderedPartitionedKVOutput.class.getName();
    356      edgeValues.inputClassName = OrderedGroupedKVInput.class.getName();
    357
    358      edgeValues.movementType = EdgeProperty.DataMovementType.SCATTER_GATHER;
    359      edgeValues.sourceType = EdgeProperty.DataSourceType.PERSISTED;
    360      edgeValues.schedulingType = EdgeProperty.SchedulingType.SEQUENTIAL;
    361      }
    362    else
    363      {
    364      addComparators( edgeValues.config, "cascading.sort.comparator", group.getSortingSelectors(), edgeValues.getResolvedSortFieldsMap().get( ordinal ) );
    365
    366      edgeValues.outputClassName = OrderedPartitionedKVOutput.class.getName();
    367      edgeValues.inputClassName = OrderedGroupedKVInput.class.getName();
    368
    369      edgeValues.movementType = EdgeProperty.DataMovementType.SCATTER_GATHER;
    370      edgeValues.sourceType = EdgeProperty.DataSourceType.PERSISTED;
    371      edgeValues.schedulingType = EdgeProperty.SchedulingType.SEQUENTIAL;
    372      }
    373
    374    if( group.isSorted() )
    375      {
    376      edgeValues.keyClassName = TuplePair.class.getName();
    377      edgeValues.keyPartitionerClassName = GroupingSortingPartitioner.class.getName();
    378
    379      if( group.isSortReversed() )
    380        edgeValues.keyComparatorClassName = ReverseGroupingSortingComparator.class.getName();
    381      else
    382        edgeValues.keyComparatorClassName = GroupingSortingComparator.class.getName();
    383      }
    384
    385    return edgeValues;
    386    }
    387
    388  private EdgeProperty createEdgeProperty( EdgeValues edgeValues )
    389    {
    390    TezConfiguration outputConfig = new TezConfiguration( edgeValues.getConfig() );
    391    outputConfig.set( "cascading.node.sink", FlowElements.id( edgeValues.getFlowElement() ) );
    392    outputConfig.set( "cascading.node.ordinals", Util.join( edgeValues.getOrdinals(), "," ) );
    393    addFields( outputConfig, "cascading.node.key.fields", edgeValues.getResolvedKeyFieldsMap() );
    394    addFields( outputConfig, "cascading.node.sort.fields", edgeValues.getResolvedSortFieldsMap() );
    395    addFields( outputConfig, "cascading.node.value.fields", edgeValues.getResolvedValueFieldsMap() );
    396
    397    UserPayload outputPayload = createIntermediatePayloadOutput( outputConfig, edgeValues );
    398
    399    TezConfiguration inputConfig = new TezConfiguration( edgeValues.getConfig() );
    400    inputConfig.set( "cascading.node.source", FlowElements.id( edgeValues.getFlowElement() ) );
    401    inputConfig.set( "cascading.node.ordinals", Util.join( edgeValues.getOrdinals(), "," ) );
    402    addFields( inputConfig, "cascading.node.key.fields", edgeValues.getResolvedKeyFieldsMap() );
    403    addFields( inputConfig, "cascading.node.sort.fields", edgeValues.getResolvedSortFieldsMap() );
    404    addFields( inputConfig, "cascading.node.value.fields", edgeValues.getResolvedValueFieldsMap() );
    405
    406    UserPayload inputPayload = createIntermediatePayloadInput( inputConfig, edgeValues );
    407
    408    return EdgeProperty.create(
    409      edgeValues.getMovementType(),
    410      edgeValues.getSourceType(),
    411      edgeValues.getSchedulingType(),
    412      OutputDescriptor.create( edgeValues.getOutputClassName() ).setUserPayload( outputPayload ),
    413      InputDescriptor.create( edgeValues.getInputClassName() ).setUserPayload( inputPayload )
    414    );
    415    }
    416
    417  private UserPayload createIntermediatePayloadOutput( TezConfiguration config, EdgeValues edgeValues )
    418    {
    419    config.set( TezRuntimeConfiguration.TEZ_RUNTIME_KEY_CLASS, edgeValues.keyClassName );
    420    config.set( TezRuntimeConfiguration.TEZ_RUNTIME_VALUE_CLASS, edgeValues.valueClassName );
    421    config.set( TezRuntimeConfiguration.TEZ_RUNTIME_KEY_COMPARATOR_CLASS, edgeValues.keyComparatorClassName );
    422    config.set( TezRuntimeConfiguration.TEZ_RUNTIME_PARTITIONER_CLASS, edgeValues.keyPartitionerClassName );
    423
    424    setWorkingDirectory( config );
    425
    426    return getPayload( config );
    427    }
    428
    429  private UserPayload createIntermediatePayloadInput( TezConfiguration config, EdgeValues edgeValues )
    430    {
    431    config.set( TezRuntimeConfiguration.TEZ_RUNTIME_KEY_CLASS, edgeValues.keyClassName );
    432    config.set( TezRuntimeConfiguration.TEZ_RUNTIME_VALUE_CLASS, edgeValues.valueClassName );
    433    config.set( TezRuntimeConfiguration.TEZ_RUNTIME_KEY_COMPARATOR_CLASS, edgeValues.keyComparatorClassName );
    434    config.set( TezRuntimeConfiguration.TEZ_RUNTIME_PARTITIONER_CLASS, edgeValues.keyPartitionerClassName );
    435
    436    setWorkingDirectory( config );
    437
    438    return getPayload( config );
    439    }
    440
    441  private static void setWorkingDirectory( Configuration conf )
    442    {
    443    String name = conf.get( JobContext.WORKING_DIR );
    444
    445    if( name != null )
    446      return;
    447
    448    try
    449      {
    450      Path dir = FileSystem.get( conf ).getWorkingDirectory();
    451      conf.set( JobContext.WORKING_DIR, dir.toString() );
    452      }
    453    catch( IOException exception )
    454      {
    455      throw new RuntimeException( exception );
    456      }
    457    }
    458
    459  public Vertex createVertex( FlowProcess<TezConfiguration> flowProcess, TezConfiguration initializedConfig, FlowNode flowNode )
    460    {
    461    JobConf conf = new JobConf( initializedConfig );
    462
    463    addInputOutputMapping( conf, flowNode );
    464
    465    conf.setBoolean( "mapred.used.genericoptionsparser", true );
    466
    467    Map<String, LocalResource> taskLocalResources = new HashMap<>();
    468
    469    Map<FlowElement, Configuration> sourceConfigs = initFromSources( flowNode, flowProcess, conf, taskLocalResources );
    470    Map<FlowElement, Configuration> sinkConfigs = initFromSinks( flowNode, flowProcess, conf );
    471
    472    initFromTraps( flowNode, flowProcess, conf );
    473
    474    initFromNodeConfigDef( flowNode, conf );
    475
    476    // force step to local mode if any tap is local
    477    setLocalMode( initializedConfig, conf, null );
    478
    479    conf.set( "cascading.flow.node.num", Integer.toString( flowNode.getOrdinal() ) );
    480
    481    HadoopUtil.setIsInflow( conf ); // must be called after all taps configurations have been retrieved
    482
    483    int parallelism = getParallelism( flowNode, conf );
    484
    485    if( parallelism == 0 )
    486      throw new FlowException( getName(), "the default number of gather partitions must be set, see cascading.flow.FlowRuntimeProps" );
    487
    488    flowNode.addProcessAnnotation( FlowRuntimeProps.GATHER_PARTITIONS, Integer.toString( parallelism ) );
    489
    490    Vertex vertex = newVertex( flowNode, conf, parallelism );
    491
    492    if( !taskLocalResources.isEmpty() )
    493      vertex.addTaskLocalFiles( taskLocalResources );
    494
    495    for( FlowElement flowElement : sourceConfigs.keySet() )
    496      {
    497      if( !( flowElement instanceof Tap ) )
    498        continue;
    499
    500      Configuration sourceConf = sourceConfigs.get( flowElement );
    501
    502      // not setting the new-api value could result in failures if not set by the Scheme
    503      if( sourceConf.get( "mapred.mapper.new-api" ) == null )
    504        HadoopUtil.setNewApi( sourceConf, sourceConf.get( "mapred.input.format.class", sourceConf.get( "mapreduce.job.inputformat.class" ) ) );
    505
    506      // unfortunately we cannot just load the input format and set it on the builder with also pulling all other
    507      // values out of the configuration.
    508      MRInput.MRInputConfigBuilder configBuilder = MRInput.createConfigBuilder( sourceConf, null );
    509
    510      // the default in Tez is true, this overrides
    511      if( conf.get( FlowRuntimeProps.COMBINE_SPLITS ) != null )
    512        configBuilder.groupSplits( conf.getBoolean( FlowRuntimeProps.COMBINE_SPLITS, true ) );
    513
    514      // grouping splits loses file name info, breaking partition tap default impl
    515      if( flowElement instanceof PartitionTap ) // todo: generify
    516        configBuilder.groupSplits( false );
    517
    518      DataSourceDescriptor dataSourceDescriptor = configBuilder.build();
    519
    520      vertex.addDataSource( FlowElements.id( flowElement ), dataSourceDescriptor );
    521      }
    522
    523    for( FlowElement flowElement : sinkConfigs.keySet() )
    524      {
    525      if( !( flowElement instanceof Tap ) )
    526        continue;
    527
    528      Configuration sinkConf = sinkConfigs.get( flowElement );
    529
    530      Class outputFormatClass;
    531      String outputPath;
    532
    533      // we have to set sane defaults if not set by the tap
    534      // typically the case of MultiSinkTap
    535      String formatClassName = sinkConf.get( "mapred.output.format.class", sinkConf.get( "mapreduce.job.outputformat.class" ) );
    536
    537      if( formatClassName == null )
    538        {
    539        outputFormatClass = TextOutputFormat.class; // unused, use "new" api, its the default
    540        outputPath = Hfs.getTempPath( sinkConf ).toString(); // unused
    541        }
    542      else
    543        {
    544        outputFormatClass = Util.loadClass( formatClassName );
    545        outputPath = getOutputPath( sinkConf );
    546        }
    547
    548      if( outputPath == null && getOutputPath( sinkConf ) == null && isFileOutputFormat( outputFormatClass ) )
    549        outputPath = Hfs.getTempPath( sinkConf ).toString(); // unused
    550
    551      MROutput.MROutputConfigBuilder configBuilder = MROutput.createConfigBuilder( sinkConf, outputFormatClass, outputPath );
    552
    553      DataSinkDescriptor dataSinkDescriptor = configBuilder.build();
    554
    555      vertex.addDataSink( FlowElements.id( flowElement ), dataSinkDescriptor );
    556      }
    557
    558    addRemoteDebug( flowNode, vertex );
    559    addRemoteProfiling( flowNode, vertex );
    560
    561    if( vertex.getTaskLaunchCmdOpts() != null )
    562      flowNode.addProcessAnnotation( TezConfiguration.TEZ_TASK_LAUNCH_CMD_OPTS, vertex.getTaskLaunchCmdOpts() );
    563
    564    return vertex;
    565    }
    566
    567  protected String getOutputPath( Configuration sinkConf )
    568    {
    569    return sinkConf.get( "mapred.output.dir", sinkConf.get( "mapreduce.output.fileoutputformat.outputdir" ) );
    570    }
    571
    572  protected boolean isFileOutputFormat( Class outputFormatClass )
    573    {
    574    return org.apache.hadoop.mapred.FileOutputFormat.class.isAssignableFrom( outputFormatClass ) ||
    575      org.apache.hadoop.mapreduce.lib.output.FileOutputFormat.class.isAssignableFrom( outputFormatClass );
    576    }
    577
    578  protected int getParallelism( FlowNode flowNode, JobConf conf )
    579    {
    580    // only count streamed taps, accumulated taps are always annotated
    581    HashSet<Tap> sourceStreamedTaps = new HashSet<>( flowNode.getSourceTaps() );
    582
    583    sourceStreamedTaps.removeAll( flowNode.getSourceElements( StreamMode.Accumulated ) );
    584
    585    if( sourceStreamedTaps.size() != 0 )
    586      return -1;
    587
    588    int parallelism = Integer.MAX_VALUE;
    589
    590    for( Tap tap : flowNode.getSinkTaps() )
    591      {
    592      int numSinkParts = tap.getScheme().getNumSinkParts();
    593
    594      if( numSinkParts == 0 )
    595        continue;
    596
    597      if( parallelism != Integer.MAX_VALUE )
    598        LOG.info( "multiple sink taps in flow node declaring numSinkParts, choosing lowest value. see cascading.flow.FlowRuntimeProps for broader control." );
    599
    600      parallelism = Math.min( parallelism, numSinkParts );
    601      }
    602
    603    if( parallelism != Integer.MAX_VALUE )
    604      return parallelism;
    605
    606    return conf.getInt( FlowRuntimeProps.GATHER_PARTITIONS, 0 );
    607    }
    608
    609  private void addInputOutputMapping( JobConf conf, FlowNode flowNode )
    610    {
    611    FlowNodeGraph flowNodeGraph = getFlowNodeGraph();
    612    Set<ProcessEdge> incomingEdges = flowNodeGraph.incomingEdgesOf( flowNode );
    613
    614    for( ProcessEdge processEdge : incomingEdges )
    615      conf.set( "cascading.node.source." + processEdge.getFlowElementID(), processEdge.getSourceProcessID() );
    616
    617    Set<ProcessEdge> outgoingEdges = flowNodeGraph.outgoingEdgesOf( flowNode );
    618
    619    for( ProcessEdge processEdge : outgoingEdges )
    620      conf.set( "cascading.node.sink." + processEdge.getFlowElementID(), processEdge.getSinkProcessID() );
    621    }
    622
    623  protected Map<FlowElement, Configuration> initFromSources( FlowNode flowNode, FlowProcess<TezConfiguration> flowProcess,
    624                                                             Configuration conf, Map<String, LocalResource> taskLocalResources )
    625    {
    626    Set<? extends FlowElement> accumulatedSources = flowNode.getSourceElements( StreamMode.Accumulated );
    627
    628    for( FlowElement element : accumulatedSources )
    629      {
    630      if( element instanceof Tap )
    631        {
    632        JobConf current = new JobConf( conf );
    633        Tap tap = (Tap) element;
    634
    635        if( tap.getIdentifier() == null )
    636          throw new IllegalStateException( "tap may not have null identifier: " + tap.toString() );
    637
    638        tap.sourceConfInit( flowProcess, current );
    639
    640        Collection<String> paths = current.getStringCollection( CASCADING_LOCAL_RESOURCES + Tap.id( tap ) );
    641
    642        if( !paths.isEmpty() )
    643          {
    644          String flowStagingPath = ( (Hadoop2TezFlow) getFlow() ).getFlowStagingPath();
    645          String resourceSubPath = Tap.id( tap );
    646          Map<Path, Path> pathMap = TezUtil.addToClassPath( current, flowStagingPath, resourceSubPath, paths, LocalResourceType.FILE, taskLocalResources, null );
    647
    648          current.setStrings( CASCADING_REMOTE_RESOURCES + Tap.id( tap ), taskLocalResources.keySet().toArray( new String[ taskLocalResources.size() ] ) );
    649
    650          allLocalResources.putAll( taskLocalResources );
    651          syncPaths.putAll( pathMap );
    652          }
    653
    654        Map<String, String> map = flowProcess.diffConfigIntoMap( new TezConfiguration( conf ), new TezConfiguration( current ) );
    655        conf.set( "cascading.node.accumulated.source.conf." + Tap.id( tap ), pack( map, conf ) );
    656
    657        setLocalMode( conf, current, tap );
    658        }
    659      }
    660
    661    Set<FlowElement> sources = new HashSet<>( flowNode.getSourceElements() );
    662
    663    sources.removeAll( accumulatedSources );
    664
    665    if( sources.isEmpty() )
    666      throw new IllegalStateException( "all sources marked as accumulated" );
    667
    668    Map<FlowElement, Configuration> configs = new HashMap<>();
    669
    670    for( FlowElement element : sources )
    671      {
    672      JobConf current = new JobConf( conf );
    673
    674      String id = FlowElements.id( element );
    675
    676      current.set( "cascading.node.source", id );
    677
    678      if( element instanceof Tap )
    679        {
    680        Tap tap = (Tap) element;
    681
    682        if( tap.getIdentifier() == null )
    683          throw new IllegalStateException( "tap may not have null identifier: " + tap.toString() );
    684
    685        tap.sourceConfInit( flowProcess, current );
    686
    687        setLocalMode( conf, current, tap );
    688        }
    689
    690      configs.put( element, current );
    691      }
    692
    693    return configs;
    694    }
    695
    696  protected Map<FlowElement, Configuration> initFromSinks( FlowNode flowNode, FlowProcess<? extends Configuration> flowProcess, Configuration conf )
    697    {
    698    Set<FlowElement> sinks = flowNode.getSinkElements();
    699    Map<FlowElement, Configuration> configs = new HashMap<>();
    700
    701    for( FlowElement element : sinks )
    702      {
    703      JobConf current = new JobConf( conf );
    704
    705      if( element instanceof Tap )
    706        {
    707        Tap tap = (Tap) element;
    708
    709        if( tap.getIdentifier() == null )
    710          throw new IllegalStateException( "tap may not have null identifier: " + element.toString() );
    711
    712        tap.sinkConfInit( flowProcess, current );
    713
    714        setLocalMode( conf, current, tap );
    715        }
    716
    717      String id = FlowElements.id( element );
    718
    719      current.set( "cascading.node.sink", id );
    720
    721      configs.put( element, current );
    722      }
    723
    724    return configs;
    725    }
    726
    727  private void initFromNodeConfigDef( FlowNode flowNode, Configuration conf )
    728    {
    729    initConfFromNodeConfigDef( flowNode.getElementGraph(), new ConfigurationSetter( conf ) );
    730    }
    731
    732  private void initFromStepConfigDef( Configuration conf )
    733    {
    734    initConfFromStepConfigDef( new ConfigurationSetter( conf ) );
    735    }
    736
    737  protected void initFromTraps( FlowNode flowNode, FlowProcess<? extends Configuration> flowProcess, Configuration conf )
    738    {
    739    Map<String, Tap> traps = flowNode.getTrapMap();
    740
    741    if( !traps.isEmpty() )
    742      {
    743      JobConf trapConf = new JobConf( conf );
    744
    745      for( Tap tap : traps.values() )
    746        {
    747        tap.sinkConfInit( flowProcess, trapConf );
    748        setLocalMode( conf, trapConf, tap );
    749        }
    750      }
    751    }
    752
    753  private Vertex newVertex( FlowNode flowNode, Configuration conf, int parallelism )
    754    {
    755    conf.set( FlowNode.CASCADING_FLOW_NODE, pack( flowNode, conf ) ); // todo: pack into payload directly
    756
    757    ProcessorDescriptor descriptor = ProcessorDescriptor.create( FlowProcessor.class.getName() );
    758
    759    descriptor.setUserPayload( getPayload( conf ) );
    760
    761    Vertex vertex = Vertex.create( flowNode.getID(), descriptor, parallelism );
    762
    763    if( environment != null )
    764      vertex.setTaskEnvironment( environment );
    765
    766    return vertex;
    767    }
    768
    769  private UserPayload getPayload( Configuration conf )
    770    {
    771    try
    772      {
    773      return TezUtils.createUserPayloadFromConf( conf );
    774      }
    775    catch( IOException exception )
    776      {
    777      throw new CascadingException( exception );
    778      }
    779    }
    780
    781  private String pack( Object object, Configuration conf )
    782    {
    783    try
    784      {
    785      return serializeBase64( object, conf, true );
    786      }
    787    catch( IOException exception )
    788      {
    789      throw new FlowException( "unable to pack object: " + object.getClass().getCanonicalName(), exception );
    790      }
    791    }
    792
    793  @Override
    794  public void clean( TezConfiguration config )
    795    {
    796    for( Tap sink : getSinkTaps() )
    797      {
    798      if( sink.isTemporary() && ( getFlow().getFlowStats().isSuccessful() || getFlow().getRunID() == null ) )
    799        {
    800        try
    801          {
    802          sink.deleteResource( config );
    803          }
    804        catch( Exception exception )
    805          {
    806          // sink all exceptions, don't fail app
    807          logWarn( "unable to remove temporary file: " + sink, exception );
    808          }
    809        }
    810      else
    811        {
    812        cleanTapMetaData( config, sink );
    813        }
    814      }
    815
    816    for( Tap tap : getTraps() )
    817      cleanTapMetaData( config, tap );
    818    }
    819
    820  private void cleanTapMetaData( TezConfiguration config, Tap tap )
    821    {
    822    try
    823      {
    824      Hadoop18TapUtil.cleanupTapMetaData( config, tap );
    825      }
    826    catch( IOException exception )
    827      {
    828      // ignore exception
    829      }
    830    }
    831
    832  public void syncArtifacts()
    833    {
    834    // this may not be strictly necessary, but there is a condition where setting the access time
    835    // fails, so there may be one were setting the modification time fails. if so, we can compensate.
    836    Map<String, Long> timestamps = HadoopUtil.syncPaths( getConfig(), syncPaths, true );
    837
    838    for( Map.Entry<String, Long> entry : timestamps.entrySet() )
    839      {
    840      LocalResource localResource = allLocalResources.get( entry.getKey() );
    841
    842      if( localResource != null )
    843        localResource.setTimestamp( entry.getValue() );
    844      }
    845    }
    846
    847  private void setLocalMode( Configuration parent, JobConf current, Tap tap )
    848    {
    849    // force step to local mode
    850    if( !HadoopUtil.isLocal( current ) )
    851      return;
    852
    853    if( tap != null )
    854      logInfo( "tap forcing step to tez local mode: " + tap.getIdentifier() );
    855
    856    HadoopUtil.setLocal( parent );
    857    }
    858
    859  private void addRemoteDebug( FlowNode flowNode, Vertex vertex )
    860    {
    861    String value = System.getProperty( "test.debug.node", null );
    862
    863    if( Util.isEmpty( value ) )
    864      return;
    865
    866    if( !flowNode.getSourceElementNames().contains( value ) && asInt( value ) != flowNode.getOrdinal() )
    867      return;
    868
    869    LOG.warn( "remote debugging enabled with property: {}, on node: {}, with node id: {}", "test.debug.node", value, flowNode.getID() );
    870
    871    String opts = vertex.getTaskLaunchCmdOpts();
    872
    873    if( opts == null )
    874      opts = "";
    875
    876    String address = System.getProperty( "test.debug.address", "localhost:5005" ).trim();
    877
    878    opts += " -agentlib:jdwp=transport=dt_socket,server=n,address=" + address + ",suspend=y";
    879
    880    vertex.setTaskLaunchCmdOpts( opts );
    881    }
    882
    883  private void addRemoteProfiling( FlowNode flowNode, Vertex vertex )
    884    {
    885    String value = System.getProperty( "test.profile.node", null );
    886
    887    if( Util.isEmpty( value ) )
    888      return;
    889
    890    if( !flowNode.getSourceElementNames().contains( value ) && asInt( value ) != flowNode.getOrdinal() )
    891      return;
    892
    893    LOG.warn( "remote profiling enabled with property: {}, on node: {}, with node id: {}", "test.profile.node", value, flowNode.getID() );
    894
    895    String opts = vertex.getTaskLaunchCmdOpts();
    896
    897    if( opts == null )
    898      opts = "";
    899
    900    String path = System.getProperty( "test.profile.path", "/tmp/jfr/" );
    901
    902    if( !path.endsWith( "/" ) )
    903      path += "/";
    904
    905    LOG.warn( "remote profiling property: {}, logging to path: {}", "test.profile.path", path );
    906
    907    opts += String.format( " -XX:+UnlockCommercialFeatures -XX:+FlightRecorder -XX:FlightRecorderOptions=defaultrecording=true,dumponexit=true,dumponexitpath=%1$s%2$s,disk=true,repository=%1$s%2$s", path, flowNode.getID() );
    908
    909    vertex.setTaskLaunchCmdOpts( opts );
    910    }
    911
    912  private int asInt( String value )
    913    {
    914    try
    915      {
    916      return Integer.parseInt( value );
    917      }
    918    catch( NumberFormatException exception )
    919      {
    920      return -1;
    921      }
    922    }
    923
    924  public Map<String, LocalResource> getAllLocalResources()
    925    {
    926    return allLocalResources;
    927    }
    928
    929  private static class EdgeValues
    930    {
    931    FlowElement flowElement;
    932    TezConfiguration config;
    933    Set<Integer> ordinals;
    934    String keyClassName;
    935    String valueClassName;
    936    String keyComparatorClassName;
    937    String keyPartitionerClassName;
    938    String outputClassName;
    939    String inputClassName;
    940    EdgeProperty.DataMovementType movementType;
    941    EdgeProperty.DataSourceType sourceType;
    942    EdgeProperty.SchedulingType schedulingType;
    943
    944    Map<Integer, Fields> resolvedKeyFieldsMap;
    945    Map<Integer, Fields> resolvedSortFieldsMap;
    946    Map<Integer, Fields> resolvedValueFieldsMap;
    947
    948    private EdgeValues( TezConfiguration config, ProcessEdge processEdge )
    949      {
    950      this.config = config;
    951      this.flowElement = processEdge.getFlowElement();
    952      this.ordinals = processEdge.getSourceProvidedOrdinals();
    953
    954      this.resolvedKeyFieldsMap = processEdge.getResolvedKeyFields();
    955      this.resolvedSortFieldsMap = processEdge.getResolvedSortFields();
    956      this.resolvedValueFieldsMap = processEdge.getResolvedValueFields();
    957      }
    958
    959    public FlowElement getFlowElement()
    960      {
    961      return flowElement;
    962      }
    963
    964    public TezConfiguration getConfig()
    965      {
    966      return config;
    967      }
    968
    969    public Set getOrdinals()
    970      {
    971      return ordinals;
    972      }
    973
    974    public String getKeyClassName()
    975      {
    976      return keyClassName;
    977      }
    978
    979    public String getValueClassName()
    980      {
    981      return valueClassName;
    982      }
    983
    984    public String getKeyComparatorClassName()
    985      {
    986      return keyComparatorClassName;
    987      }
    988
    989    public String getKeyPartitionerClassName()
    990      {
    991      return keyPartitionerClassName;
    992      }
    993
    994    public String getOutputClassName()
    995      {
    996      return outputClassName;
    997      }
    998
    999    public String getInputClassName()
    1000      {
    1001      return inputClassName;
    1002      }
    1003
    1004    public EdgeProperty.DataMovementType getMovementType()
    1005      {
    1006      return movementType;
    1007      }
    1008
    1009    public EdgeProperty.DataSourceType getSourceType()
    1010      {
    1011      return sourceType;
    1012      }
    1013
    1014    public EdgeProperty.SchedulingType getSchedulingType()
    1015      {
    1016      return schedulingType;
    1017      }
    1018
    1019    public Map<Integer, Fields> getResolvedKeyFieldsMap()
    1020      {
    1021      return resolvedKeyFieldsMap;
    1022      }
    1023
    1024    public Map<Integer, Fields> getResolvedSortFieldsMap()
    1025      {
    1026      return resolvedSortFieldsMap;
    1027      }
    1028
    1029    public Map<Integer, Fields> getResolvedValueFieldsMap()
    1030      {
    1031      return resolvedValueFieldsMap;
    1032      }
    1033    }
    1034  }