cascading.flow.hadoop.util

## Class HadoopUtil

- java.lang.Object
- - cascading.flow.hadoop.util.HadoopUtil

    public class HadoopUtil
    extends java.lang.Object

### Field Summary

Fields
`static java.lang.String` `CASCADING_FLOW_EXECUTING`

### Constructor Summary

Constructors
Constructor and Description
`HadoopUtil()`

### Method Summary

Methods
`static void` `addComparators(Configuration conf, java.lang.String property, java.util.Map<java.lang.String,Fields> map, BaseFlowStep flowStep, Group group)`
`static void` `addComparators(Configuration conf, java.lang.String property, java.util.Map<java.lang.String,Fields> map, Fields resolvedFields)`
`static void` `addFields(Configuration conf, java.lang.String property, java.util.Map<java.lang.Integer,Fields> fields)`
`static void` `addInputPath(Configuration conf, Path path)`
`static void` `addInputPaths(Configuration conf, java.lang.Iterable<Path> paths)`
`static JobConf` `asJobConfInstance(Configuration configuration)`
`protected static <C extends Configuration> C` `callCopyConstructor(java.lang.Class type, Configuration parent)`
`static <C> C` `copyConfiguration(C parent)`
`static <C extends Configuration> C` `copyConfiguration(java.util.Map<java.lang.Object,java.lang.Object> srcProperties, C dstConfiguration)`
`static <C> C` `copyJobConf(C parentJobConf)`
`static JobConf` `copyJobConf(JobConf parentJobConf)`
`static JobConf` `createJobConf(java.util.Map<java.lang.Object,java.lang.Object> properties, JobConf defaultJobconf)`
`static PlatformInfo` `createPlatformInfo(java.lang.Class type, java.lang.String attributePath, java.lang.String platformName)`
`static java.util.Map<java.lang.Object,java.lang.Object>` `createProperties(Configuration jobConf)`
`static byte[]` `decodeBytes(java.lang.String string)`
`static <T> T` `deserializeBase64(java.lang.String string, Configuration conf, java.lang.Class<T> type)` This method deserializes the Base64 encoded String into an Object instance.
`static <T> T` `deserializeBase64(java.lang.String string, Configuration conf, java.lang.Class<T> type, boolean decompress)`
`static java.lang.String` `encodeBytes(byte[] bytes)`
`static java.lang.Class` `findMainClass(java.lang.Class defaultType)`
`static java.util.Map<Path,Path>` `getCommonPaths(java.util.Map<java.lang.String,Path> localPaths, java.util.Map<java.lang.String,Path> remotePaths)`
`static java.util.Map<java.lang.String,java.lang.String>` `getConfig(Configuration defaultConf, Configuration updatedConf)`
`static FileSystem` `getDefaultFS(Configuration config)`
`static java.util.Map<java.lang.Integer,Fields>` `getFields(Configuration conf, java.lang.String property)`
`static java.lang.Thread` `getHDFSShutdownHook()`
`static JobConf[]` `getJobConfs(Configuration job, java.util.List<java.util.Map<java.lang.String,java.lang.String>> configs)`
`static LocalFileSystem` `getLocalFS(Configuration config)`
`static Path` `getOutputPath(Configuration conf)`
`static PlatformInfo` `getPlatformInfo()`
`static PlatformInfo` `getPlatformInfo(java.lang.Class type, java.lang.String attributePath, java.lang.String platformName)`
`static void` `initLog4j(Configuration configuration)`
`static void` `initLog4j(JobConf configuration)`
`static <T> ObjectSerializer` `instantiateSerializer(Configuration conf, java.lang.Class<T> type)`
`static boolean` `isInflow(Configuration conf)`
`static boolean` `isLocal(Configuration conf)`
`static boolean` `isYARN(Configuration conf)`
`static <J extends Configuration> J` `mergeConf(J job, java.util.Map<java.lang.String,java.lang.String> config, boolean directly)`
`static java.lang.String` `pack(java.lang.Object object, Configuration conf)`
`static Configuration` `removePropertiesFrom(Configuration jobConf, java.lang.String... keys)`
`static boolean` `removeStateFromDistCache(Configuration conf, java.lang.String path)`
`static void` `resolvePaths(Configuration config, java.util.Collection<java.lang.String> classpath, java.lang.String remoteRoot, java.lang.String resourceSubPath, java.util.Map<java.lang.String,Path> localPaths, java.util.Map<java.lang.String,Path> remotePaths)`
`static <T> java.lang.String` `serializeBase64(T object, Configuration conf)`
`static <T> java.lang.String` `serializeBase64(T object, Configuration conf, boolean compress)`
`static void` `setIsInflow(Configuration conf)`
`static void` `setLocal(Configuration conf)`
`static boolean` `setNewApi(Configuration conf, java.lang.String className)`
`static void` `setOutputPath(Configuration conf, Path path)`
`static java.util.Map<java.lang.String,java.lang.Long>` `syncPaths(Configuration config, java.util.Map<Path,Path> commonPaths, boolean syncTimes)` Copies paths from one local path to a remote path.

### Methods inherited from class java.lang.Object

`clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait`

### Field Detail

#### CASCADING_FLOW_EXECUTING

    public static final java.lang.String CASCADING_FLOW_EXECUTING

See Also:
  Constant Field Values

### Constructor Detail

#### HadoopUtil

    public HadoopUtil()

### Method Detail

#### setIsInflow

    public static void setIsInflow(Configuration conf)

#### isInflow

    public static boolean isInflow(Configuration conf)

#### initLog4j

    public static void initLog4j(JobConf configuration)

#### initLog4j

    public static void initLog4j(Configuration configuration)

#### asJobConfInstance

    public static JobConf asJobConfInstance(Configuration configuration)

#### copyJobConf

    public static <C> C copyJobConf(C parentJobConf)

#### copyJobConf

    public static JobConf copyJobConf(JobConf parentJobConf)

#### createJobConf

    public static JobConf createJobConf(java.util.Map<java.lang.Object,java.lang.Object> properties,
                        JobConf defaultJobconf)

#### copyConfiguration

    public static <C> C copyConfiguration(C parent)

#### callCopyConstructor

    protected static <C extends Configuration> C callCopyConstructor(java.lang.Class type,
                                                  Configuration parent)

#### copyConfiguration

    public static <C extends Configuration> C copyConfiguration(java.util.Map<java.lang.Object,java.lang.Object> srcProperties,
                                                C dstConfiguration)

#### createProperties

    public static java.util.Map<java.lang.Object,java.lang.Object> createProperties(Configuration jobConf)

#### getHDFSShutdownHook

    public static java.lang.Thread getHDFSShutdownHook()

#### encodeBytes

    public static java.lang.String encodeBytes(byte[] bytes)

#### decodeBytes

    public static byte[] decodeBytes(java.lang.String string)

#### instantiateSerializer

    public static <T> ObjectSerializer instantiateSerializer(Configuration conf,
                                             java.lang.Class<T> type)
                                                  throws java.lang.ClassNotFoundException

Throws:
  `java.lang.ClassNotFoundException`

#### serializeBase64

    public static <T> java.lang.String serializeBase64(T object,
                                       Configuration conf)
                                            throws java.io.IOException

Throws:
  `java.io.IOException`

#### serializeBase64

    public static <T> java.lang.String serializeBase64(T object,
                                       Configuration conf,
                                       boolean compress)
                                            throws java.io.IOException

Throws:
  `java.io.IOException`

#### deserializeBase64

    public static <T> T deserializeBase64(java.lang.String string,
                          Configuration conf,
                          java.lang.Class<T> type)
                               throws java.io.IOException

This method deserializes the Base64 encoded String into an Object instance.

Parameters:
  `string` -
Returns:
  an Object
Throws:
  `java.io.IOException`

#### deserializeBase64

    public static <T> T deserializeBase64(java.lang.String string,
                          Configuration conf,
                          java.lang.Class<T> type,
                          boolean decompress)
                               throws java.io.IOException

Throws:
  `java.io.IOException`

#### findMainClass

    public static java.lang.Class findMainClass(java.lang.Class defaultType)

#### getConfig

    public static java.util.Map<java.lang.String,java.lang.String> getConfig(Configuration defaultConf,
                                                             Configuration updatedConf)

#### getJobConfs

    public static JobConf[] getJobConfs(Configuration job,
                        java.util.List<java.util.Map<java.lang.String,java.lang.String>> configs)

#### mergeConf

    public static <J extends Configuration> J mergeConf(J job,
                                        java.util.Map<java.lang.String,java.lang.String> config,
                                        boolean directly)

#### removePropertiesFrom

    public static Configuration removePropertiesFrom(Configuration jobConf,
                                     java.lang.String... keys)

#### removeStateFromDistCache

    public static boolean removeStateFromDistCache(Configuration conf,
                                   java.lang.String path)
                                            throws java.io.IOException

Throws:
  `java.io.IOException`

#### getPlatformInfo

    public static PlatformInfo getPlatformInfo()

#### getPlatformInfo

    public static PlatformInfo getPlatformInfo(java.lang.Class type,
                               java.lang.String attributePath,
                               java.lang.String platformName)

#### createPlatformInfo

    public static PlatformInfo createPlatformInfo(java.lang.Class type,
                                  java.lang.String attributePath,
                                  java.lang.String platformName)

#### syncPaths

    public static java.util.Map<java.lang.String,java.lang.Long> syncPaths(Configuration config,
                                                           java.util.Map<Path,Path> commonPaths,
                                                           boolean syncTimes)

Copies paths from one local path to a remote path. If syncTimes is true, both modification and access time are changed to match the local 'from' path.

Returns a map of file-name to remote modification times if the remote time is different than the local time.

Parameters:
  `config` -
  `commonPaths` -
  `syncTimes` -

#### getCommonPaths

    public static java.util.Map<Path,Path> getCommonPaths(java.util.Map<java.lang.String,Path> localPaths,
                                          java.util.Map<java.lang.String,Path> remotePaths)

#### resolvePaths

    public static void resolvePaths(Configuration config,
                    java.util.Collection<java.lang.String> classpath,
                    java.lang.String remoteRoot,
                    java.lang.String resourceSubPath,
                    java.util.Map<java.lang.String,Path> localPaths,
                    java.util.Map<java.lang.String,Path> remotePaths)

#### getLocalFS

    public static LocalFileSystem getLocalFS(Configuration config)

#### getDefaultFS

    public static FileSystem getDefaultFS(Configuration config)

#### isLocal

    public static boolean isLocal(Configuration conf)

#### isYARN

    public static boolean isYARN(Configuration conf)

#### setLocal

    public static void setLocal(Configuration conf)

#### setNewApi

    public static boolean setNewApi(Configuration conf,
                    java.lang.String className)

#### addInputPaths

    public static void addInputPaths(Configuration conf,
                     java.lang.Iterable<Path> paths)

#### addInputPath

    public static void addInputPath(Configuration conf,
                    Path path)

#### setOutputPath

    public static void setOutputPath(Configuration conf,
                     Path path)

#### getOutputPath

    public static Path getOutputPath(Configuration conf)

#### pack

    public static java.lang.String pack(java.lang.Object object,
                        Configuration conf)

#### addFields

    public static void addFields(Configuration conf,
                 java.lang.String property,
                 java.util.Map<java.lang.Integer,Fields> fields)

#### getFields

    public static java.util.Map<java.lang.Integer,Fields> getFields(Configuration conf,
                                                    java.lang.String property)
                                                             throws java.io.IOException

Throws:
  `java.io.IOException`

#### addComparators

    public static void addComparators(Configuration conf,
                      java.lang.String property,
                      java.util.Map<java.lang.String,Fields> map,
                      BaseFlowStep flowStep,
                      Group group)

#### addComparators

    public static void addComparators(Configuration conf,
                      java.lang.String property,
                      java.util.Map<java.lang.String,Fields> map,
                      Fields resolvedFields)