cascading.flow.hadoop.util

## Class HadoopUtil

- java.lang.Object
- - cascading.flow.hadoop.util.HadoopUtil

    public class HadoopUtil
    extends Object

### Field Summary

Fields
`static String` `CASCADING_FLOW_EXECUTING`

### Constructor Summary

Constructors
Constructor and Description
`HadoopUtil()`

### Method Summary

Methods
`static void` `addComparators(Configuration conf, String property, Map<String,Fields> map, BaseFlowStep flowStep, Group group)`
`static void` `addComparators(Configuration conf, String property, Map<String,Fields> map, Fields resolvedFields)`
`static void` `addFields(Configuration conf, String property, Map<Integer,Fields> fields)`
`static void` `addInputPath(Configuration conf, Path path)`
`static JobConf` `asJobConfInstance(Configuration configuration)`
`protected static <C extends Configuration> C` `callCopyConstructor(Class type, Configuration parent)`
`static <C> C` `copyConfiguration(C parent)`
`static <C extends Configuration> C` `copyConfiguration(Map<Object,Object> srcProperties, C dstConfiguration)`
`static <C> C` `copyJobConf(C parentJobConf)`
`static JobConf` `copyJobConf(JobConf parentJobConf)`
`static JobConf` `createJobConf(Map<Object,Object> properties, JobConf defaultJobconf)`
`static PlatformInfo` `createPlatformInfo(Class type, String attributePath, String platformName)`
`static Map<Object,Object>` `createProperties(Configuration jobConf)`
`static byte[]` `decodeBytes(String string)`
`static <T> T` `deserializeBase64(String string, Configuration conf, Class<T> type)` This method deserializes the Base64 encoded String into an Object instance.
`static <T> T` `deserializeBase64(String string, Configuration conf, Class<T> type, boolean decompress)`
`static String` `encodeBytes(byte[] bytes)`
`static Class` `findMainClass(Class defaultType)`
`static Map<Path,Path>` `getCommonPaths(Map<String,Path> localPaths, Map<String,Path> remotePaths)`
`static Map<String,String>` `getConfig(Configuration defaultConf, Configuration updatedConf)`
`static FileSystem` `getDefaultFS(Configuration config)`
`static Map<Integer,Fields>` `getFields(Configuration conf, String property)`
`static Thread` `getHDFSShutdownHook()`
`static JobConf[]` `getJobConfs(Configuration job, List<Map<String,String>> configs)`
`static LocalFileSystem` `getLocalFS(Configuration config)`
`static Path` `getOutputPath(Configuration conf)`
`static PlatformInfo` `getPlatformInfo()`
`static PlatformInfo` `getPlatformInfo(Class type, String attributePath, String platformName)`
`static void` `initLog4j(Configuration configuration)`
`static void` `initLog4j(JobConf configuration)`
`static <T> ObjectSerializer` `instantiateSerializer(Configuration conf, Class<T> type)`
`static boolean` `isInflow(Configuration conf)`
`static boolean` `isLocal(Configuration conf)`
`static boolean` `isYARN(Configuration conf)`
`static <J extends Configuration> J` `mergeConf(J job, Map<String,String> config, boolean directly)`
`static String` `pack(Object object, Configuration conf)`
`static Configuration` `removePropertiesFrom(Configuration jobConf, String... keys)`
`static boolean` `removeStateFromDistCache(Configuration conf, String path)`
`static void` `resolvePaths(Configuration config, Collection<String> classpath, String remoteRoot, String resourceSubPath, Map<String,Path> localPaths, Map<String,Path> remotePaths)`
`static <T> String` `serializeBase64(T object, Configuration conf)`
`static <T> String` `serializeBase64(T object, Configuration conf, boolean compress)`
`static void` `setIsInflow(Configuration conf)`
`static void` `setLocal(Configuration conf)`
`static boolean` `setNewApi(Configuration conf, String className)`
`static void` `setOutputPath(Configuration conf, Path path)`
`static Map<String,Long>` `syncPaths(Configuration config, Map<Path,Path> commonPaths, boolean syncTimes)` Copies paths from one local path to a remote path.

### Methods inherited from class java.lang.Object

`clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait`

### Field Detail

#### CASCADING_FLOW_EXECUTING

    public static final String CASCADING_FLOW_EXECUTING

See Also:
  Constant Field Values

### Constructor Detail

#### HadoopUtil

    public HadoopUtil()

### Method Detail

#### setIsInflow

    public static void setIsInflow(Configuration conf)

#### isInflow

    public static boolean isInflow(Configuration conf)

#### initLog4j

    public static void initLog4j(JobConf configuration)

#### initLog4j

    public static void initLog4j(Configuration configuration)

#### asJobConfInstance

    public static JobConf asJobConfInstance(Configuration configuration)

#### copyJobConf

    public static <C> C copyJobConf(C parentJobConf)

#### copyJobConf

    public static JobConf copyJobConf(JobConf parentJobConf)

#### createJobConf

    public static JobConf createJobConf(Map<Object,Object> properties,
                        JobConf defaultJobconf)

#### copyConfiguration

    public static <C> C copyConfiguration(C parent)

#### callCopyConstructor

    protected static <C extends Configuration> C callCopyConstructor(Class type,
                                                  Configuration parent)

#### copyConfiguration

    public static <C extends Configuration> C copyConfiguration(Map<Object,Object> srcProperties,
                                                C dstConfiguration)

#### createProperties

    public static Map<Object,Object> createProperties(Configuration jobConf)

#### getHDFSShutdownHook

    public static Thread getHDFSShutdownHook()

#### encodeBytes

    public static String encodeBytes(byte[] bytes)

#### decodeBytes

    public static byte[] decodeBytes(String string)

#### instantiateSerializer

    public static <T> ObjectSerializer instantiateSerializer(Configuration conf,
                                             Class<T> type)
                                                  throws ClassNotFoundException

Throws:
  `ClassNotFoundException`

#### serializeBase64

    public static <T> String serializeBase64(T object,
                             Configuration conf)
                                  throws IOException

Throws:
  `IOException`

#### serializeBase64

    public static <T> String serializeBase64(T object,
                             Configuration conf,
                             boolean compress)
                                  throws IOException

Throws:
  `IOException`

#### deserializeBase64

    public static <T> T deserializeBase64(String string,
                          Configuration conf,
                          Class<T> type)
                               throws IOException

This method deserializes the Base64 encoded String into an Object instance.

Parameters:
  `string` -
Returns:
  an Object
Throws:
  `IOException`

#### deserializeBase64

    public static <T> T deserializeBase64(String string,
                          Configuration conf,
                          Class<T> type,
                          boolean decompress)
                               throws IOException

Throws:
  `IOException`

#### findMainClass

    public static Class findMainClass(Class defaultType)

#### getConfig

    public static Map<String,String> getConfig(Configuration defaultConf,
                               Configuration updatedConf)

#### getJobConfs

    public static JobConf[] getJobConfs(Configuration job,
                        List<Map<String,String>> configs)

#### mergeConf

    public static <J extends Configuration> J mergeConf(J job,
                                        Map<String,String> config,
                                        boolean directly)

#### removePropertiesFrom

    public static Configuration removePropertiesFrom(Configuration jobConf,
                                     String... keys)

#### removeStateFromDistCache

    public static boolean removeStateFromDistCache(Configuration conf,
                                   String path)
                                            throws IOException

Throws:
  `IOException`

#### getPlatformInfo

    public static PlatformInfo getPlatformInfo()

#### getPlatformInfo

    public static PlatformInfo getPlatformInfo(Class type,
                               String attributePath,
                               String platformName)

#### createPlatformInfo

    public static PlatformInfo createPlatformInfo(Class type,
                                  String attributePath,
                                  String platformName)

#### syncPaths

    public static Map<String,Long> syncPaths(Configuration config,
                             Map<Path,Path> commonPaths,
                             boolean syncTimes)

Copies paths from one local path to a remote path. If syncTimes is true, both modification and access time are changed to match the local 'from' path.

Returns a map of file-name to remote modification times if the remote time is different than the local time.

Parameters:
  `config` -
  `commonPaths` -
  `syncTimes` -

#### getCommonPaths

    public static Map<Path,Path> getCommonPaths(Map<String,Path> localPaths,
                                Map<String,Path> remotePaths)

#### resolvePaths

    public static void resolvePaths(Configuration config,
                    Collection<String> classpath,
                    String remoteRoot,
                    String resourceSubPath,
                    Map<String,Path> localPaths,
                    Map<String,Path> remotePaths)

#### getLocalFS

    public static LocalFileSystem getLocalFS(Configuration config)

#### getDefaultFS

    public static FileSystem getDefaultFS(Configuration config)

#### isLocal

    public static boolean isLocal(Configuration conf)

#### isYARN

    public static boolean isYARN(Configuration conf)

#### setLocal

    public static void setLocal(Configuration conf)

#### setNewApi

    public static boolean setNewApi(Configuration conf,
                    String className)

#### addInputPath

    public static void addInputPath(Configuration conf,
                    Path path)

#### setOutputPath

    public static void setOutputPath(Configuration conf,
                     Path path)

#### getOutputPath

    public static Path getOutputPath(Configuration conf)

#### pack

    public static String pack(Object object,
              Configuration conf)

#### addFields

    public static void addFields(Configuration conf,
                 String property,
                 Map<Integer,Fields> fields)

#### getFields

    public static Map<Integer,Fields> getFields(Configuration conf,
                                String property)
                                         throws IOException

Throws:
  `IOException`

#### addComparators

    public static void addComparators(Configuration conf,
                      String property,
                      Map<String,Fields> map,
                      BaseFlowStep flowStep,
                      Group group)

#### addComparators

    public static void addComparators(Configuration conf,
                      String property,
                      Map<String,Fields> map,
                      Fields resolvedFields)