cascading.flow.hadoop.util
Class HadoopUtil
java.lang.Object
cascading.flow.hadoop.util.HadoopUtil
public class HadoopUtil
- extends java.lang.Object
Method Summary |
static JobConf |
createJobConf(java.util.Map<java.lang.Object,java.lang.Object> properties,
JobConf defaultJobconf)
|
static java.util.Map<java.lang.Object,java.lang.Object> |
createProperties(Configuration jobConf)
|
static java.lang.Object |
deserializeBase64(java.lang.String string)
This method deserializes the Base64 encoded String into an Object instance. |
static java.lang.Object |
deserializeBase64(java.lang.String string,
boolean decompress)
|
static java.util.List<java.util.Map<java.lang.String,java.lang.String>> |
deserializeListMapBase64(java.lang.String string,
boolean decompress)
|
static java.util.Map<java.lang.String,java.lang.String> |
deserializeMapBase64(java.lang.String string,
boolean decompress)
|
static java.lang.Class |
findMainClass(java.lang.Class defaultType)
|
static java.util.Map<java.lang.String,java.lang.String> |
getConfig(JobConf defaultConf,
JobConf updatedConf)
|
static java.lang.Thread |
getHDFSShutdownHook()
|
static JobConf[] |
getJobConfs(JobConf job,
java.util.List<java.util.Map<java.lang.String,java.lang.String>> configs)
|
static void |
initLog4j(JobConf jobConf)
|
static JobConf |
mergeConf(JobConf job,
java.util.Map<java.lang.String,java.lang.String> config,
boolean directly)
|
static java.lang.String |
readStateFromDistCache(JobConf jobConf,
java.lang.String id)
|
static JobConf |
removePropertiesFrom(JobConf jobConf,
java.lang.String... keys)
|
static boolean |
removeStateFromDistCache(JobConf conf,
java.lang.String path)
|
static java.lang.String |
serializeBase64(java.lang.Object object)
This method serializes the given Object instance and retunrs a String Base64 representation. |
static java.lang.String |
serializeBase64(java.lang.Object object,
boolean compress)
|
static java.lang.String |
serializeListMapBase64(java.util.List<java.util.Map<java.lang.String,java.lang.String>> listMap,
boolean compress)
|
static java.lang.String |
serializeMapBase64(java.util.Map<java.lang.String,java.lang.String> map,
boolean compress)
|
static java.lang.String |
writeStateToDistCache(JobConf conf,
java.lang.String id,
java.lang.String stepState)
|
Methods inherited from class java.lang.Object |
clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait |
HadoopUtil
public HadoopUtil()
initLog4j
public static void initLog4j(JobConf jobConf)
createJobConf
public static JobConf createJobConf(java.util.Map<java.lang.Object,java.lang.Object> properties,
JobConf defaultJobconf)
createProperties
public static java.util.Map<java.lang.Object,java.lang.Object> createProperties(Configuration jobConf)
getHDFSShutdownHook
public static java.lang.Thread getHDFSShutdownHook()
serializeMapBase64
public static java.lang.String serializeMapBase64(java.util.Map<java.lang.String,java.lang.String> map,
boolean compress)
throws java.io.IOException
- Throws:
java.io.IOException
deserializeMapBase64
public static java.util.Map<java.lang.String,java.lang.String> deserializeMapBase64(java.lang.String string,
boolean decompress)
throws java.io.IOException
- Throws:
java.io.IOException
serializeListMapBase64
public static java.lang.String serializeListMapBase64(java.util.List<java.util.Map<java.lang.String,java.lang.String>> listMap,
boolean compress)
throws java.io.IOException
- Throws:
java.io.IOException
deserializeListMapBase64
public static java.util.List<java.util.Map<java.lang.String,java.lang.String>> deserializeListMapBase64(java.lang.String string,
boolean decompress)
throws java.io.IOException
- Throws:
java.io.IOException
serializeBase64
public static java.lang.String serializeBase64(java.lang.Object object)
throws java.io.IOException
- This method serializes the given Object instance and retunrs a String Base64 representation.
- Parameters:
object
- to be serialized
- Returns:
- String
- Throws:
java.io.IOException
serializeBase64
public static java.lang.String serializeBase64(java.lang.Object object,
boolean compress)
throws java.io.IOException
- Throws:
java.io.IOException
deserializeBase64
public static java.lang.Object deserializeBase64(java.lang.String string)
throws java.io.IOException
- This method deserializes the Base64 encoded String into an Object instance.
- Parameters:
string
-
- Returns:
- an Object
- Throws:
java.io.IOException
deserializeBase64
public static java.lang.Object deserializeBase64(java.lang.String string,
boolean decompress)
throws java.io.IOException
- Throws:
java.io.IOException
findMainClass
public static java.lang.Class findMainClass(java.lang.Class defaultType)
getConfig
public static java.util.Map<java.lang.String,java.lang.String> getConfig(JobConf defaultConf,
JobConf updatedConf)
getJobConfs
public static JobConf[] getJobConfs(JobConf job,
java.util.List<java.util.Map<java.lang.String,java.lang.String>> configs)
mergeConf
public static JobConf mergeConf(JobConf job,
java.util.Map<java.lang.String,java.lang.String> config,
boolean directly)
removePropertiesFrom
public static JobConf removePropertiesFrom(JobConf jobConf,
java.lang.String... keys)
removeStateFromDistCache
public static boolean removeStateFromDistCache(JobConf conf,
java.lang.String path)
throws java.io.IOException
- Throws:
java.io.IOException
writeStateToDistCache
public static java.lang.String writeStateToDistCache(JobConf conf,
java.lang.String id,
java.lang.String stepState)
readStateFromDistCache
public static java.lang.String readStateFromDistCache(JobConf jobConf,
java.lang.String id)
throws java.io.IOException
- Throws:
java.io.IOException
Copyright © 2007-2011 Concurrent, Inc. All Rights Reserved.