001/*
002 * Copyright (c) 2016-2017 Chris K Wensel <chris@wensel.net>. All Rights Reserved.
003 * Copyright (c) 2007-2017 Xplenty, Inc. All Rights Reserved.
004 *
005 * Project and contact information: http://www.cascading.org/
006 *
007 * This file is part of the Cascading project.
008 *
009 * Licensed under the Apache License, Version 2.0 (the "License");
010 * you may not use this file except in compliance with the License.
011 * You may obtain a copy of the License at
012 *
013 *     http://www.apache.org/licenses/LICENSE-2.0
014 *
015 * Unless required by applicable law or agreed to in writing, software
016 * distributed under the License is distributed on an "AS IS" BASIS,
017 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
018 * See the License for the specific language governing permissions and
019 * limitations under the License.
020 */
021
022package cascading;
023
024import java.io.IOException;
025import java.util.HashMap;
026import java.util.HashSet;
027import java.util.Map;
028import java.util.Set;
029
030import cascading.flow.FlowConnectorProps;
031import cascading.operation.DebugLevel;
032import cascading.platform.PlatformRunner;
033import cascading.platform.TestPlatform;
034import org.junit.After;
035import org.junit.Before;
036import org.junit.runner.RunWith;
037import org.slf4j.Logger;
038import org.slf4j.LoggerFactory;
039
040/**
041 * PlatformTestCase is the base class for JUnit tests that are platform agnostic. That is using the {@link TestPlatform}
042 * interface each test can be run against all supported platform like Hadoop or Cascading local mode.
043 * <p>
044 * It is strongly recommended users look at the source of {@link FieldedPipesPlatformTest} or related tests to see how
045 * this class is used.
046 * <p>
047 * This test case uses the {@link PlatformRunner} to inject the available platform providers which implement the
048 * TestPlatform base class.
049 * <p>
050 * By default the PlatformRunner looks for "cascading/platform/platform.properties" file in the classpath, and
051 * instantiates the class specified by the "platform.classname" property. If more than one "platform.properties"
052 * resource is found, each class is instantiated and the whole suite of tests will be run against each instance.
053 * <p>
054 * To limit this, setting the system property "platform.includes" to list the platform names that should be run will
055 * cause the PlatformRunner to ignore any unlisted platforms. Thus setting {@code platform.includes=local}, only
056 * local mode will run even if the "hadoop" platform was found in the classpath.
057 * <p>
058 * To pass custom properties to each test to be used by the {@link cascading.flow.FlowConnector}, create
059 * system properties prefixed by "platform.". These properties, minus the "platform." prefix in the property name,
060 * will override any defaults.
061 * <p>
062 * Subclasses of PlatformTestCase can set "{@code useCluster} to {@code true} on the constructor if the underlying
063 * platform can boot a cluster for testing. By setting the system property "test.cluster.enabled" to false, this
064 * can be deactivated in order to temporarily speed test execution. By default {@code useCluster} is {@code false},
065 * typically user tests don't need to have a cluster running to test their functionality so leaving the default is
066 * reasonable.
067 */
068@RunWith(PlatformRunner.class)
069public abstract class PlatformTestCase extends CascadingTestCase
070  {
071  private static final Logger LOG = LoggerFactory.getLogger( PlatformTestCase.class );
072
073  static Set<String> allPaths = new HashSet<String>();
074
075  Set<String> currentPaths = new HashSet<String>();
076
077  private transient TestPlatform platform = null;
078
079  private transient boolean useCluster;
080  private transient int numMapTasks;
081  private transient int numGatherPartitions;
082
083  protected PlatformTestCase( boolean useCluster )
084    {
085    this.useCluster = useCluster;
086    }
087
088  protected PlatformTestCase( boolean useCluster, int numMapTasks, int numGatherPartitions )
089    {
090    this( useCluster );
091    this.numMapTasks = numMapTasks;
092    this.numGatherPartitions = numGatherPartitions;
093    }
094
095  protected PlatformTestCase()
096    {
097    this( false );
098    }
099
100  public void installPlatform( TestPlatform platform )
101    {
102    this.platform = platform;
103    this.platform.setUseCluster( useCluster );
104
105    if( this.platform.isMapReduce() )
106      {
107      platform.setNumMappers( numMapTasks );
108      platform.setNumReducers( numGatherPartitions );
109      }
110
111    if( this.platform.isDAG() )
112      platform.setNumGatherPartitions( numGatherPartitions );
113    }
114
115  public TestPlatform getPlatform()
116    {
117    return platform;
118    }
119
120  @Override
121  protected String[] getOutputPathElements()
122    {
123    return new String[]{getTestOutputRoot(), getPlatformName(), getTestCaseName(), getTestName()};
124    }
125
126  @Override
127  protected String[] getPlanPathElements()
128    {
129    return new String[]{getTestPlanRoot(), getPlatformName(), getTestCaseName(), getTestName()};
130    }
131
132  public String getOutputPath( String path )
133    {
134    String result = makeOutputPath( path );
135
136    if( allPaths.contains( result ) )
137      throw new IllegalStateException( "path already has been used:" + result );
138
139    allPaths.add( result );
140    currentPaths.add( result );
141
142    return result;
143    }
144
145  protected String makeOutputPath( String path )
146    {
147    if( path.startsWith( "/" ) )
148      return getOutputPath() + path;
149
150    return getOutputPath() + "/" + path;
151    }
152
153  public String getPlatformName()
154    {
155    return platform.getName();
156    }
157
158  @Before
159  public void setUp() throws Exception
160    {
161    super.setUp();
162    getPlatform().setUp();
163    }
164
165  public Map<Object, Object> getProperties()
166    {
167    return new HashMap<Object, Object>( getPlatform().getProperties() );
168    }
169
170  protected void copyFromLocal( String inputFile ) throws IOException
171    {
172    getPlatform().copyFromLocal( inputFile );
173    }
174
175  protected Map<Object, Object> disableDebug()
176    {
177    Map<Object, Object> properties = getProperties();
178    FlowConnectorProps.setDebugLevel( properties, DebugLevel.NONE );
179
180    return properties;
181    }
182
183  @After
184  public void tearDown() throws Exception
185    {
186    try
187      {
188      for( String path : currentPaths )
189        {
190        LOG.info( "copying to local {}", path );
191
192        if( getPlatform().isUseCluster() && getPlatform().remoteExists( path ) )
193          getPlatform().copyToLocal( path );
194        }
195
196      currentPaths.clear();
197      }
198    finally
199      {
200      getPlatform().tearDown();
201      }
202    }
203  }