001/* 002 * Copyright (c) 2016-2017 Chris K Wensel <chris@wensel.net>. All Rights Reserved. 003 * Copyright (c) 2007-2017 Xplenty, Inc. All Rights Reserved. 004 * 005 * Project and contact information: http://www.cascading.org/ 006 * 007 * This file is part of the Cascading project. 008 * 009 * Licensed under the Apache License, Version 2.0 (the "License"); 010 * you may not use this file except in compliance with the License. 011 * You may obtain a copy of the License at 012 * 013 * http://www.apache.org/licenses/LICENSE-2.0 014 * 015 * Unless required by applicable law or agreed to in writing, software 016 * distributed under the License is distributed on an "AS IS" BASIS, 017 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 018 * See the License for the specific language governing permissions and 019 * limitations under the License. 020 */ 021 022package cascading.flow.hadoop2; 023 024import java.beans.ConstructorProperties; 025import java.util.Map; 026 027import cascading.flow.FlowConnector; 028import cascading.flow.hadoop.planner.MapReduceHadoopRuleRegistry; 029import cascading.flow.planner.FlowPlanner; 030import cascading.flow.planner.rule.RuleRegistrySet; 031import cascading.scheme.Scheme; 032import cascading.scheme.hadoop.SequenceFile; 033 034/** 035 * Use the Hadoop2MR1FlowConnector to link source and sink {@link cascading.tap.Tap} instances with an assembly of {@link cascading.pipe.Pipe} instances into 036 * an executable {@link cascading.flow.hadoop.HadoopFlow} for execution on an Apache Hadoop cluster. 037 * 038 * @see cascading.property.AppProps 039 * @see cascading.flow.FlowConnectorProps 040 * @see cascading.flow.FlowDef 041 * @see cascading.flow.hadoop.MapReduceFlow 042 */ 043public class Hadoop2MR1FlowConnector extends FlowConnector 044 { 045 /** 046 * Constructor FlowConnector creates a new FlowConnector instance. 047 * <p> 048 * All properties passed to Hadoop are retrieved from a default instantiation of the Hadoop 049 * {@link org.apache.hadoop.mapred.JobConf} which pulls all properties from the local CLASSPATH. 050 */ 051 public Hadoop2MR1FlowConnector() 052 { 053 } 054 055 /** 056 * Constructor FlowConnector creates a new FlowConnector instance using the given {@link java.util.Properties} instance as 057 * default value for the underlying jobs. All properties are copied to a new native configuration instance. 058 * 059 * @param properties of type Properties 060 */ 061 @ConstructorProperties({"properties"}) 062 public Hadoop2MR1FlowConnector( Map<Object, Object> properties ) 063 { 064 super( properties ); 065 } 066 067 /** 068 * Constructor HadoopFlowConnector creates a new HadoopFlowConnector instance. 069 * <p> 070 * All properties passed to Hadoop are retrieved from a default instantiation of the Hadoop 071 * {@link org.apache.hadoop.mapred.JobConf} which pulls all properties from the local CLASSPATH. 072 * 073 * @param ruleRegistrySet of type RuleRegistry 074 */ 075 @ConstructorProperties({"ruleRegistrySet"}) 076 public Hadoop2MR1FlowConnector( RuleRegistrySet ruleRegistrySet ) 077 { 078 super( ruleRegistrySet ); 079 } 080 081 /** 082 * Constructor HadoopFlowConnector creates a new HadoopFlowConnector instance using the given {@link java.util.Properties} instance as 083 * default value for the underlying jobs. All properties are copied to a new native configuration instance. 084 * 085 * @param properties of type Map 086 * @param ruleRegistrySet of type RuleRegistry 087 */ 088 @ConstructorProperties({"properties", "ruleRegistrySet"}) 089 public Hadoop2MR1FlowConnector( Map<Object, Object> properties, RuleRegistrySet ruleRegistrySet ) 090 { 091 super( properties, ruleRegistrySet ); 092 } 093 094 @Override 095 protected Class<? extends Scheme> getDefaultIntermediateSchemeClass() 096 { 097 return SequenceFile.class; 098 } 099 100 @Override 101 protected FlowPlanner createFlowPlanner() 102 { 103 return new Hadoop2MR1Planner(); 104 } 105 106 @Override 107 protected RuleRegistrySet createDefaultRuleRegistrySet() 108 { 109 return new RuleRegistrySet( new MapReduceHadoopRuleRegistry() ); 110 } 111 }