001/* 002 * Copyright (c) 2016-2017 Chris K Wensel <chris@wensel.net>. All Rights Reserved. 003 * Copyright (c) 2007-2017 Xplenty, Inc. All Rights Reserved. 004 * 005 * Project and contact information: http://www.cascading.org/ 006 * 007 * This file is part of the Cascading project. 008 * 009 * Licensed under the Apache License, Version 2.0 (the "License"); 010 * you may not use this file except in compliance with the License. 011 * You may obtain a copy of the License at 012 * 013 * http://www.apache.org/licenses/LICENSE-2.0 014 * 015 * Unless required by applicable law or agreed to in writing, software 016 * distributed under the License is distributed on an "AS IS" BASIS, 017 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 018 * See the License for the specific language governing permissions and 019 * limitations under the License. 020 */ 021 022package cascading; 023 024import java.io.IOException; 025import java.util.HashMap; 026import java.util.HashSet; 027import java.util.Map; 028import java.util.Set; 029 030import cascading.flow.FlowConnectorProps; 031import cascading.operation.DebugLevel; 032import cascading.platform.PlatformRunner; 033import cascading.platform.TestPlatform; 034import org.junit.After; 035import org.junit.Before; 036import org.junit.runner.RunWith; 037import org.slf4j.Logger; 038import org.slf4j.LoggerFactory; 039 040/** 041 * PlatformTestCase is the base class for JUnit tests that are platform agnostic. That is using the {@link TestPlatform} 042 * interface each test can be run against all supported platform like Hadoop or Cascading local mode. 043 * <p> 044 * It is strongly recommended users look at the source of {@link FieldedPipesPlatformTest} or related tests to see how 045 * this class is used. 046 * <p> 047 * This test case uses the {@link PlatformRunner} to inject the available platform providers which implement the 048 * TestPlatform base class. 049 * <p> 050 * By default the PlatformRunner looks for "cascading/platform/platform.properties" file in the classpath, and 051 * instantiates the class specified by the "platform.classname" property. If more than one "platform.properties" 052 * resource is found, each class is instantiated and the whole suite of tests will be run against each instance. 053 * <p> 054 * To limit this, setting the system property "platform.includes" to list the platform names that should be run will 055 * cause the PlatformRunner to ignore any unlisted platforms. Thus setting {@code platform.includes=local}, only 056 * local mode will run even if the "hadoop" platform was found in the classpath. 057 * <p> 058 * To pass custom properties to each test to be used by the {@link cascading.flow.FlowConnector}, create 059 * system properties prefixed by "platform.". These properties, minus the "platform." prefix in the property name, 060 * will override any defaults. 061 * <p> 062 * Subclasses of PlatformTestCase can set "{@code useCluster} to {@code true} on the constructor if the underlying 063 * platform can boot a cluster for testing. By setting the system property "test.cluster.enabled" to false, this 064 * can be deactivated in order to temporarily speed test execution. By default {@code useCluster} is {@code false}, 065 * typically user tests don't need to have a cluster running to test their functionality so leaving the default is 066 * reasonable. 067 */ 068@RunWith(PlatformRunner.class) 069public abstract class PlatformTestCase extends CascadingTestCase 070 { 071 private static final Logger LOG = LoggerFactory.getLogger( PlatformTestCase.class ); 072 073 static Set<String> allPaths = new HashSet<String>(); 074 075 Set<String> currentPaths = new HashSet<String>(); 076 077 private transient TestPlatform platform = null; 078 079 private transient boolean useCluster; 080 private transient int numMapTasks; 081 private transient int numGatherPartitions; 082 083 protected PlatformTestCase( boolean useCluster ) 084 { 085 this.useCluster = useCluster; 086 } 087 088 protected PlatformTestCase( boolean useCluster, int numMapTasks, int numGatherPartitions ) 089 { 090 this( useCluster ); 091 this.numMapTasks = numMapTasks; 092 this.numGatherPartitions = numGatherPartitions; 093 } 094 095 protected PlatformTestCase() 096 { 097 this( false ); 098 } 099 100 public void installPlatform( TestPlatform platform ) 101 { 102 this.platform = platform; 103 this.platform.setUseCluster( useCluster ); 104 105 if( this.platform.isMapReduce() ) 106 { 107 platform.setNumMappers( numMapTasks ); 108 platform.setNumReducers( numGatherPartitions ); 109 } 110 111 if( this.platform.isDAG() ) 112 platform.setNumGatherPartitions( numGatherPartitions ); 113 } 114 115 public TestPlatform getPlatform() 116 { 117 return platform; 118 } 119 120 @Override 121 protected String[] getOutputPathElements() 122 { 123 return new String[]{getTestOutputRoot(), getPlatformName(), getTestCaseName(), getTestName()}; 124 } 125 126 @Override 127 protected String[] getPlanPathElements() 128 { 129 return new String[]{getTestPlanRoot(), getPlatformName(), getTestCaseName(), getTestName()}; 130 } 131 132 public String getOutputPath( String path ) 133 { 134 String result = makeOutputPath( path ); 135 136 if( allPaths.contains( result ) ) 137 throw new IllegalStateException( "path already has been used:" + result ); 138 139 allPaths.add( result ); 140 currentPaths.add( result ); 141 142 return result; 143 } 144 145 protected String makeOutputPath( String path ) 146 { 147 if( path.startsWith( "/" ) ) 148 return getOutputPath() + path; 149 150 return getOutputPath() + "/" + path; 151 } 152 153 public String getPlatformName() 154 { 155 return platform.getName(); 156 } 157 158 @Before 159 public void setUp() throws Exception 160 { 161 super.setUp(); 162 getPlatform().setUp(); 163 } 164 165 public Map<Object, Object> getProperties() 166 { 167 return new HashMap<Object, Object>( getPlatform().getProperties() ); 168 } 169 170 protected void copyFromLocal( String inputFile ) throws IOException 171 { 172 getPlatform().copyFromLocal( inputFile ); 173 } 174 175 protected Map<Object, Object> disableDebug() 176 { 177 Map<Object, Object> properties = getProperties(); 178 FlowConnectorProps.setDebugLevel( properties, DebugLevel.NONE ); 179 180 return properties; 181 } 182 183 @After 184 public void tearDown() throws Exception 185 { 186 try 187 { 188 for( String path : currentPaths ) 189 { 190 LOG.info( "copying to local {}", path ); 191 192 if( getPlatform().isUseCluster() && getPlatform().remoteExists( path ) ) 193 getPlatform().copyToLocal( path ); 194 } 195 196 currentPaths.clear(); 197 } 198 finally 199 { 200 getPlatform().tearDown(); 201 } 202 } 203 }