cascading.flow.hadoop.util

## Class HadoopUtil

- java.lang.Object
- - cascading.flow.hadoop.util.HadoopUtil

-     public class HadoopUtil
      extends java.lang.Object

- - ### Field Summary

    Fields
    Modifier and Type | Field and Description
    --- | ---
    `static java.lang.String` | `CASCADING_FLOW_EXECUTING`

  - ### Constructor Summary

    Constructors
    Constructor and Description
    `HadoopUtil()`

  - ### Method Summary

    Methods
    Modifier and Type | Method and Description
    --- | ---
    `static void` | `addComparators(org.apache.hadoop.conf.Configuration conf, java.lang.String property, java.util.Map<java.lang.String,Fields> map, BaseFlowStep flowStep, Group group)`
    `static void` | `addComparators(org.apache.hadoop.conf.Configuration conf, java.lang.String property, java.util.Map<java.lang.String,Fields> map, Fields resolvedFields)`
    `static void` | `addFields(org.apache.hadoop.conf.Configuration conf, java.lang.String property, java.util.Map<java.lang.Integer,Fields> fields)`
    `static void` | `addInputPath(org.apache.hadoop.conf.Configuration conf, org.apache.hadoop.fs.Path path)`
    `static void` | `addInputPaths(org.apache.hadoop.conf.Configuration conf, java.lang.Iterable<org.apache.hadoop.fs.Path> paths)`
    `static org.apache.hadoop.mapred.JobConf` | `asJobConfInstance(org.apache.hadoop.conf.Configuration configuration)`
    `protected static <C extends org.apache.hadoop.conf.Configuration> C` | `callCopyConstructor(java.lang.Class type, org.apache.hadoop.conf.Configuration parent)`
    `static <C> C` | `copyConfiguration(C parent)`
    `static <C extends org.apache.hadoop.conf.Configuration> C` | `copyConfiguration(java.util.Map<java.lang.Object,java.lang.Object> srcProperties, C dstConfiguration)`
    `static <C> C` | `copyJobConf(C parentJobConf)`
    `static org.apache.hadoop.mapred.JobConf` | `copyJobConf(org.apache.hadoop.mapred.JobConf parentJobConf)`
    `static org.apache.hadoop.mapred.JobConf` | `createJobConf(java.util.Map<java.lang.Object,java.lang.Object> properties, org.apache.hadoop.mapred.JobConf defaultJobconf)`
    `static PlatformInfo` | `createPlatformInfo(java.lang.Class type, java.lang.String attributePath, java.lang.String platformName)`
    `static java.util.Map<java.lang.Object,java.lang.Object>` | `createProperties(org.apache.hadoop.conf.Configuration jobConf)`
    `static byte[]` | `decodeBytes(java.lang.String string)`
    `static <T> T` | `deserializeBase64(java.lang.String string, org.apache.hadoop.conf.Configuration conf, java.lang.Class<T> type)` This method deserializes the Base64 encoded String into an Object instance.
    `static <T> T` | `deserializeBase64(java.lang.String string, org.apache.hadoop.conf.Configuration conf, java.lang.Class<T> type, boolean decompress)`
    `static java.lang.String` | `encodeBytes(byte[] bytes)`
    `static java.lang.Class` | `findMainClass(java.lang.Class defaultType)`
    `static java.util.Map<org.apache.hadoop.fs.Path,org.apache.hadoop.fs.Path>` | `getCommonPaths(java.util.Map<java.lang.String,org.apache.hadoop.fs.Path> localPaths, java.util.Map<java.lang.String,org.apache.hadoop.fs.Path> remotePaths)`
    `static java.util.Map<java.lang.String,java.lang.String>` | `getConfig(org.apache.hadoop.conf.Configuration defaultConf, org.apache.hadoop.conf.Configuration updatedConf)`
    `static org.apache.hadoop.fs.FileSystem` | `getDefaultFS(org.apache.hadoop.conf.Configuration config)`
    `static java.util.Map<java.lang.Integer,Fields>` | `getFields(org.apache.hadoop.conf.Configuration conf, java.lang.String property)`
    `static java.lang.Thread` | `getHDFSShutdownHook()`
    `static org.apache.hadoop.mapred.JobConf[]` | `getJobConfs(org.apache.hadoop.conf.Configuration job, java.util.List<java.util.Map<java.lang.String,java.lang.String>> configs)`
    `static org.apache.hadoop.fs.LocalFileSystem` | `getLocalFS(org.apache.hadoop.conf.Configuration config)`
    `static org.apache.hadoop.fs.Path` | `getOutputPath(org.apache.hadoop.conf.Configuration conf)`
    `static PlatformInfo` | `getPlatformInfo()`
    `static PlatformInfo` | `getPlatformInfo(java.lang.Class type, java.lang.String attributePath, java.lang.String platformName)`
    `static void` | `initLog4j(org.apache.hadoop.conf.Configuration configuration)`
    `static void` | `initLog4j(org.apache.hadoop.mapred.JobConf configuration)`
    `static <T> ObjectSerializer` | `instantiateSerializer(org.apache.hadoop.conf.Configuration conf, java.lang.Class<T> type)`
    `static boolean` | `isInflow(org.apache.hadoop.conf.Configuration conf)`
    `static boolean` | `isLocal(org.apache.hadoop.conf.Configuration conf)`
    `static boolean` | `isYARN(org.apache.hadoop.conf.Configuration conf)`
    `static <J extends org.apache.hadoop.conf.Configuration> J` | `mergeConf(J job, java.util.Map<java.lang.String,java.lang.String> config, boolean directly)`
    `static java.lang.String` | `pack(java.lang.Object object, org.apache.hadoop.conf.Configuration conf)`
    `static org.apache.hadoop.conf.Configuration` | `removePropertiesFrom(org.apache.hadoop.conf.Configuration jobConf, java.lang.String... keys)`
    `static boolean` | `removeStateFromDistCache(org.apache.hadoop.conf.Configuration conf, java.lang.String path)`
    `static void` | `resolvePaths(org.apache.hadoop.conf.Configuration config, java.util.Collection<java.lang.String> classpath, java.lang.String remoteRoot, java.lang.String resourceSubPath, java.util.Map<java.lang.String,org.apache.hadoop.fs.Path> localPaths, java.util.Map<java.lang.String,org.apache.hadoop.fs.Path> remotePaths)`
    `static <T> java.lang.String` | `serializeBase64(T object, org.apache.hadoop.conf.Configuration conf)`
    `static <T> java.lang.String` | `serializeBase64(T object, org.apache.hadoop.conf.Configuration conf, boolean compress)`
    `static void` | `setIsInflow(org.apache.hadoop.conf.Configuration conf)`
    `static void` | `setLocal(org.apache.hadoop.conf.Configuration conf)`
    `static boolean` | `setNewApi(org.apache.hadoop.conf.Configuration conf, java.lang.String className)`
    `static void` | `setOutputPath(org.apache.hadoop.conf.Configuration conf, org.apache.hadoop.fs.Path path)`
    `static java.util.Map<java.lang.String,java.lang.Long>` | `syncPaths(org.apache.hadoop.conf.Configuration config, java.util.Map<org.apache.hadoop.fs.Path,org.apache.hadoop.fs.Path> commonPaths, boolean syncTimes)` Copies paths from one local path to a remote path.

    - ### Methods inherited from class java.lang.Object

      `clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait`

- - ### Field Detail

    - #### CASCADING_FLOW_EXECUTING

          public static final java.lang.String CASCADING_FLOW_EXECUTING

      See Also:
        Constant Field Values

  - ### Constructor Detail

    - #### HadoopUtil

          public HadoopUtil()

  - ### Method Detail

    - #### setIsInflow

          public static void setIsInflow(org.apache.hadoop.conf.Configuration conf)

    - #### isInflow

          public static boolean isInflow(org.apache.hadoop.conf.Configuration conf)

    - #### initLog4j

          public static void initLog4j(org.apache.hadoop.mapred.JobConf configuration)

    - #### initLog4j

          public static void initLog4j(org.apache.hadoop.conf.Configuration configuration)

    - #### asJobConfInstance

          public static org.apache.hadoop.mapred.JobConf asJobConfInstance(org.apache.hadoop.conf.Configuration configuration)

    - #### copyJobConf

          public static <C> C copyJobConf(C parentJobConf)

    - #### copyJobConf

          public static org.apache.hadoop.mapred.JobConf copyJobConf(org.apache.hadoop.mapred.JobConf parentJobConf)

    - #### createJobConf

          public static org.apache.hadoop.mapred.JobConf createJobConf(java.util.Map<java.lang.Object,java.lang.Object> properties,
                                                       org.apache.hadoop.mapred.JobConf defaultJobconf)

    - #### copyConfiguration

          public static <C> C copyConfiguration(C parent)

    - #### callCopyConstructor

          protected static <C extends org.apache.hadoop.conf.Configuration> C callCopyConstructor(java.lang.Class type,
                                                                               org.apache.hadoop.conf.Configuration parent)

    - #### copyConfiguration

          public static <C extends org.apache.hadoop.conf.Configuration> C copyConfiguration(java.util.Map<java.lang.Object,java.lang.Object> srcProperties,
                                                                             C dstConfiguration)

    - #### createProperties

          public static java.util.Map<java.lang.Object,java.lang.Object> createProperties(org.apache.hadoop.conf.Configuration jobConf)

    - #### getHDFSShutdownHook

          public static java.lang.Thread getHDFSShutdownHook()

    - #### encodeBytes

          public static java.lang.String encodeBytes(byte[] bytes)

    - #### decodeBytes

          public static byte[] decodeBytes(java.lang.String string)

    - #### instantiateSerializer

          public static <T> ObjectSerializer instantiateSerializer(org.apache.hadoop.conf.Configuration conf,
                                                   java.lang.Class<T> type)
                                                        throws java.lang.ClassNotFoundException

      Throws:
        `java.lang.ClassNotFoundException`

    - #### serializeBase64

          public static <T> java.lang.String serializeBase64(T object,
                                             org.apache.hadoop.conf.Configuration conf)
                                                  throws java.io.IOException

      Throws:
        `java.io.IOException`

    - #### serializeBase64

          public static <T> java.lang.String serializeBase64(T object,
                                             org.apache.hadoop.conf.Configuration conf,
                                             boolean compress)
                                                  throws java.io.IOException

      Throws:
        `java.io.IOException`

    - #### deserializeBase64

          public static <T> T deserializeBase64(java.lang.String string,
                                org.apache.hadoop.conf.Configuration conf,
                                java.lang.Class<T> type)
                                     throws java.io.IOException

      This method deserializes the Base64 encoded String into an Object instance.
      Parameters:
        `string` -
      Returns:
        an Object
      Throws:
        `java.io.IOException`

    - #### deserializeBase64

          public static <T> T deserializeBase64(java.lang.String string,
                                org.apache.hadoop.conf.Configuration conf,
                                java.lang.Class<T> type,
                                boolean decompress)
                                     throws java.io.IOException

      Throws:
        `java.io.IOException`

    - #### findMainClass

          public static java.lang.Class findMainClass(java.lang.Class defaultType)

    - #### getConfig

          public static java.util.Map<java.lang.String,java.lang.String> getConfig(org.apache.hadoop.conf.Configuration defaultConf,
                                                                   org.apache.hadoop.conf.Configuration updatedConf)

    - #### getJobConfs

          public static org.apache.hadoop.mapred.JobConf[] getJobConfs(org.apache.hadoop.conf.Configuration job,
                                                       java.util.List<java.util.Map<java.lang.String,java.lang.String>> configs)

    - #### mergeConf

          public static <J extends org.apache.hadoop.conf.Configuration> J mergeConf(J job,
                                                                     java.util.Map<java.lang.String,java.lang.String> config,
                                                                     boolean directly)

    - #### removePropertiesFrom

          public static org.apache.hadoop.conf.Configuration removePropertiesFrom(org.apache.hadoop.conf.Configuration jobConf,
                                                                  java.lang.String... keys)

    - #### removeStateFromDistCache

          public static boolean removeStateFromDistCache(org.apache.hadoop.conf.Configuration conf,
                                         java.lang.String path)
                                                  throws java.io.IOException

      Throws:
        `java.io.IOException`

    - #### getPlatformInfo

          public static PlatformInfo getPlatformInfo()

    - #### getPlatformInfo

          public static PlatformInfo getPlatformInfo(java.lang.Class type,
                                     java.lang.String attributePath,
                                     java.lang.String platformName)

    - #### createPlatformInfo

          public static PlatformInfo createPlatformInfo(java.lang.Class type,
                                        java.lang.String attributePath,
                                        java.lang.String platformName)

    - #### syncPaths

          public static java.util.Map<java.lang.String,java.lang.Long> syncPaths(org.apache.hadoop.conf.Configuration config,
                                                                 java.util.Map<org.apache.hadoop.fs.Path,org.apache.hadoop.fs.Path> commonPaths,
                                                                 boolean syncTimes)

      Copies paths from one local path to a remote path. If syncTimes is true, both modification and access time are changed to match the local 'from' path.

      Returns a map of file-name to remote modification times if the remote time is different than the local time.
      Parameters:
        `config` -
        `commonPaths` -
        `syncTimes` -

    - #### getCommonPaths

          public static java.util.Map<org.apache.hadoop.fs.Path,org.apache.hadoop.fs.Path> getCommonPaths(java.util.Map<java.lang.String,org.apache.hadoop.fs.Path> localPaths,
                                                                                          java.util.Map<java.lang.String,org.apache.hadoop.fs.Path> remotePaths)

    - #### resolvePaths

          public static void resolvePaths(org.apache.hadoop.conf.Configuration config,
                          java.util.Collection<java.lang.String> classpath,
                          java.lang.String remoteRoot,
                          java.lang.String resourceSubPath,
                          java.util.Map<java.lang.String,org.apache.hadoop.fs.Path> localPaths,
                          java.util.Map<java.lang.String,org.apache.hadoop.fs.Path> remotePaths)

    - #### getLocalFS

          public static org.apache.hadoop.fs.LocalFileSystem getLocalFS(org.apache.hadoop.conf.Configuration config)

    - #### getDefaultFS

          public static org.apache.hadoop.fs.FileSystem getDefaultFS(org.apache.hadoop.conf.Configuration config)

    - #### isLocal

          public static boolean isLocal(org.apache.hadoop.conf.Configuration conf)

    - #### isYARN

          public static boolean isYARN(org.apache.hadoop.conf.Configuration conf)

    - #### setLocal

          public static void setLocal(org.apache.hadoop.conf.Configuration conf)

    - #### setNewApi

          public static boolean setNewApi(org.apache.hadoop.conf.Configuration conf,
                          java.lang.String className)

    - #### addInputPaths

          public static void addInputPaths(org.apache.hadoop.conf.Configuration conf,
                           java.lang.Iterable<org.apache.hadoop.fs.Path> paths)

    - #### addInputPath

          public static void addInputPath(org.apache.hadoop.conf.Configuration conf,
                          org.apache.hadoop.fs.Path path)

    - #### setOutputPath

          public static void setOutputPath(org.apache.hadoop.conf.Configuration conf,
                           org.apache.hadoop.fs.Path path)

    - #### getOutputPath

          public static org.apache.hadoop.fs.Path getOutputPath(org.apache.hadoop.conf.Configuration conf)

    - #### pack

          public static java.lang.String pack(java.lang.Object object,
                              org.apache.hadoop.conf.Configuration conf)

    - #### addFields

          public static void addFields(org.apache.hadoop.conf.Configuration conf,
                       java.lang.String property,
                       java.util.Map<java.lang.Integer,Fields> fields)

    - #### getFields

          public static java.util.Map<java.lang.Integer,Fields> getFields(org.apache.hadoop.conf.Configuration conf,
                                                          java.lang.String property)
                                                                   throws java.io.IOException

      Throws:
        `java.io.IOException`

    - #### addComparators

          public static void addComparators(org.apache.hadoop.conf.Configuration conf,
                            java.lang.String property,
                            java.util.Map<java.lang.String,Fields> map,
                            BaseFlowStep flowStep,
                            Group group)

    - #### addComparators

          public static void addComparators(org.apache.hadoop.conf.Configuration conf,
                            java.lang.String property,
                            java.util.Map<java.lang.String,Fields> map,
                            Fields resolvedFields)

Copyright © 2007-2015 Xplenty, Inc. All Rights Reserved.