hadoop MRJobConfig 源码
haddop MRJobConfig 代码
文件路径：/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.mapreduce;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceAudience.Public;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.hadoop.util.Shell;
import org.apache.hadoop.yarn.util.Apps;
@InterfaceAudience.Private
@InterfaceStability.Evolving
public interface MRJobConfig {
// Used by MapTask
public static final String MAP_SORT_CLASS = "map.sort.class";
// Put all of the attribute names in here so that Job and JobContext are
// consistent.
public static final String INPUT_FORMAT_CLASS_ATTR = "mapreduce.job.inputformat.class";
public static final String MAP_CLASS_ATTR = "mapreduce.job.map.class";
public static final String MAP_OUTPUT_COLLECTOR_CLASS_ATTR
= "mapreduce.job.map.output.collector.class";
public static final String COMBINE_CLASS_ATTR = "mapreduce.job.combine.class";
public static final String REDUCE_CLASS_ATTR = "mapreduce.job.reduce.class";
public static final String OUTPUT_FORMAT_CLASS_ATTR = "mapreduce.job.outputformat.class";
public static final String PARTITIONER_CLASS_ATTR = "mapreduce.job.partitioner.class";
public static final String SETUP_CLEANUP_NEEDED = "mapreduce.job.committer.setup.cleanup.needed";
public static final String TASK_CLEANUP_NEEDED = "mapreduce.job.committer.task.cleanup.needed";
public static final String JOB_SINGLE_DISK_LIMIT_BYTES =
"mapreduce.job.local-fs.single-disk-limit.bytes";
// negative values disable the limit
public static final long DEFAULT_JOB_SINGLE_DISK_LIMIT_BYTES = -1;
public static final String JOB_DFS_STORAGE_CAPACITY_KILL_LIMIT_EXCEED =
"mapreduce.job.dfs.storage.capacity.kill-limit-exceed";
public static final boolean DEFAULT_JOB_DFS_STORAGE_CAPACITY_KILL_LIMIT_EXCEED = false;
public static final String JOB_SINGLE_DISK_LIMIT_KILL_LIMIT_EXCEED =
"mapreduce.job.local-fs.single-disk-limit.check.kill-limit-exceed";
// setting to false only logs the kill
public static final boolean DEFAULT_JOB_SINGLE_DISK_LIMIT_KILL_LIMIT_EXCEED = true;
public static final String JOB_SINGLE_DISK_LIMIT_CHECK_INTERVAL_MS =
"mapreduce.job.local-fs.single-disk-limit.check.interval-ms";
public static final long DEFAULT_JOB_SINGLE_DISK_LIMIT_CHECK_INTERVAL_MS = 5000;
public static final String TASK_LOCAL_WRITE_LIMIT_BYTES =
"mapreduce.task.local-fs.write-limit.bytes";
// negative values disable the limit
public static final long DEFAULT_TASK_LOCAL_WRITE_LIMIT_BYTES = -1;
public static final String JAR = "mapreduce.job.jar";
public static final String ID = "mapreduce.job.id";
public static final String JOB_NAME = "mapreduce.job.name";
public static final String JAR_UNPACK_PATTERN = "mapreduce.job.jar.unpack.pattern";
public static final String USER_NAME = "mapreduce.job.user.name";
public static final String PRIORITY = "mapreduce.job.priority";
public static final String QUEUE_NAME = "mapreduce.job.queuename";
/**
* Node Label expression applicable for all Job containers.
*/
public static final String JOB_NODE_LABEL_EXP = "mapreduce.job.node-label-expression";
/**
* Node Label expression applicable for AM containers.
*/
public static final String AM_NODE_LABEL_EXP = "mapreduce.job.am.node-label-expression";
/**
* Node Label expression applicable for map containers.
*/
public static final String MAP_NODE_LABEL_EXP = "mapreduce.map.node-label-expression";
/**
* Node Label expression applicable for reduce containers.
*/
public static final String REDUCE_NODE_LABEL_EXP = "mapreduce.reduce.node-label-expression";
/**
* Specify strict locality on a comma-separated list of racks and/or nodes.
* Syntax: /rack or /rack/node or node (assumes /default-rack)
*/
public static final String AM_STRICT_LOCALITY = "mapreduce.job.am.strict-locality";
public static final String RESERVATION_ID = "mapreduce.job.reservation.id";
public static final String JOB_TAGS = "mapreduce.job.tags";
public static final String JVM_NUMTASKS_TORUN = "mapreduce.job.jvm.numtasks";
public static final String SPLIT_FILE = "mapreduce.job.splitfile";
public static final String SPLIT_METAINFO_MAXSIZE = "mapreduce.job.split.metainfo.maxsize";
public static final long DEFAULT_SPLIT_METAINFO_MAXSIZE = 10000000L;
public static final String NUM_MAPS = "mapreduce.job.maps";
public static final String MAX_TASK_FAILURES_PER_TRACKER = "mapreduce.job.maxtaskfailures.per.tracker";
public static final String COMPLETED_MAPS_FOR_REDUCE_SLOWSTART = "mapreduce.job.reduce.slowstart.completedmaps";
public static final String NUM_REDUCES = "mapreduce.job.reduces";
public static final String SKIP_RECORDS = "mapreduce.job.skiprecords";
public static final String SKIP_OUTDIR = "mapreduce.job.skip.outdir";
// SPECULATIVE_SLOWNODE_THRESHOLD is obsolete and will be deleted in the future
@Deprecated
public static final String SPECULATIVE_SLOWNODE_THRESHOLD = "mapreduce.job.speculative.slownodethreshold";
public static final String SPECULATIVE_SLOWTASK_THRESHOLD = "mapreduce.job.speculative.slowtaskthreshold";
// SPECULATIVECAP is obsolete and will be deleted in the future
@Deprecated
public static final String SPECULATIVECAP = "mapreduce.job.speculative.speculativecap";
public static final String SPECULATIVECAP_RUNNING_TASKS =
"mapreduce.job.speculative.speculative-cap-running-tasks";
public static final double DEFAULT_SPECULATIVECAP_RUNNING_TASKS =
0.1;
public static final String SPECULATIVECAP_TOTAL_TASKS =
"mapreduce.job.speculative.speculative-cap-total-tasks";
public static final double DEFAULT_SPECULATIVECAP_TOTAL_TASKS =
0.01;
public static final String SPECULATIVE_MINIMUM_ALLOWED_TASKS =
"mapreduce.job.speculative.minimum-allowed-tasks";
public static final int DEFAULT_SPECULATIVE_MINIMUM_ALLOWED_TASKS =
10;
public static final String SPECULATIVE_RETRY_AFTER_NO_SPECULATE =
"mapreduce.job.speculative.retry-after-no-speculate";
public static final long DEFAULT_SPECULATIVE_RETRY_AFTER_NO_SPECULATE =
1000L;
public static final String SPECULATIVE_RETRY_AFTER_SPECULATE =
"mapreduce.job.speculative.retry-after-speculate";
public static final long DEFAULT_SPECULATIVE_RETRY_AFTER_SPECULATE =
15000L;
public static final String JOB_LOCAL_DIR = "mapreduce.job.local.dir";
public static final String OUTPUT_KEY_CLASS = "mapreduce.job.output.key.class";
public static final String OUTPUT_VALUE_CLASS = "mapreduce.job.output.value.class";
public static final String KEY_COMPARATOR = "mapreduce.job.output.key.comparator.class";
public static final String COMBINER_GROUP_COMPARATOR_CLASS = "mapreduce.job.combiner.group.comparator.class";
public static final String GROUP_COMPARATOR_CLASS = "mapreduce.job.output.group.comparator.class";
public static final String WORKING_DIR = "mapreduce.job.working.dir";
public static final String CLASSPATH_ARCHIVES = "mapreduce.job.classpath.archives";
public static final String CLASSPATH_FILES = "mapreduce.job.classpath.files";
public static final String CACHE_FILES = "mapreduce.job.cache.files";
public static final String CACHE_ARCHIVES = "mapreduce.job.cache.archives";
public static final String CACHE_FILES_SIZES = "mapreduce.job.cache.files.filesizes"; // internal use only
public static final String CACHE_ARCHIVES_SIZES = "mapreduce.job.cache.archives.filesizes"; // ditto
public static final String CACHE_LOCALFILES = "mapreduce.job.cache.local.files";
public static final String CACHE_LOCALARCHIVES = "mapreduce.job.cache.local.archives";
public static final String CACHE_FILE_TIMESTAMPS = "mapreduce.job.cache.files.timestamps";
public static final String CACHE_ARCHIVES_TIMESTAMPS = "mapreduce.job.cache.archives.timestamps";
public static final String CACHE_FILE_VISIBILITIES = "mapreduce.job.cache.files.visibilities";
public static final String CACHE_ARCHIVES_VISIBILITIES = "mapreduce.job.cache.archives.visibilities";
/**
* This parameter controls the visibility of the localized job jar on the node
* manager. If set to true, the visibility will be set to
* LocalResourceVisibility.PUBLIC. If set to false, the visibility will be set
* to LocalResourceVisibility.APPLICATION. This is a generated parameter and
* should not be set manually via config files.
*/
String JOBJAR_VISIBILITY = "mapreduce.job.jobjar.visibility";
boolean JOBJAR_VISIBILITY_DEFAULT = false;
/**
* This is a generated parameter and should not be set manually via config
* files.
*/
String JOBJAR_SHARED_CACHE_UPLOAD_POLICY =
"mapreduce.job.jobjar.sharedcache.uploadpolicy";
boolean JOBJAR_SHARED_CACHE_UPLOAD_POLICY_DEFAULT = false;
/**
* This is a generated parameter and should not be set manually via config
* files.
*/
String CACHE_FILES_SHARED_CACHE_UPLOAD_POLICIES =
"mapreduce.job.cache.files.sharedcache.uploadpolicies";
/**
* This is a generated parameter and should not be set manually via config
* files.
*/
String CACHE_ARCHIVES_SHARED_CACHE_UPLOAD_POLICIES =
"mapreduce.job.cache.archives.sharedcache.uploadpolicies";
/**
* A comma delimited list of file resources that are needed for this MapReduce
* job. These resources, if the files resource type is enabled, should either
* use the shared cache or be added to the shared cache. This parameter can be
* modified programmatically using the MapReduce Job api.
*/
String FILES_FOR_SHARED_CACHE = "mapreduce.job.cache.sharedcache.files";
/**
* A comma delimited list of libjar resources that are needed for this
* MapReduce job. These resources, if the libjars resource type is enabled,
* should either use the shared cache or be added to the shared cache. These
* resources will also be added to the classpath of all tasks for this
* MapReduce job. This parameter can be modified programmatically using the
* MapReduce Job api.
*/
String FILES_FOR_CLASSPATH_AND_SHARED_CACHE =
"mapreduce.job.cache.sharedcache.files.addtoclasspath";
/**
* A comma delimited list of archive resources that are needed for this
* MapReduce job. These resources, if the archives resource type is enabled,
* should either use the shared cache or be added to the shared cache. This
* parameter can be modified programmatically using the MapReduce Job api.
*/
String ARCHIVES_FOR_SHARED_CACHE =
"mapreduce.job.cache.sharedcache.archives";
/**
* A comma delimited list of resource categories that are enabled for the
* shared cache. If a category is enabled, resources in that category will be
* uploaded to the shared cache. The valid categories are: jobjar, libjars,
* files, archives. If "disabled" is specified then all categories are
* disabled. If "enabled" is specified then all categories are enabled.
*/
String SHARED_CACHE_MODE = "mapreduce.job.sharedcache.mode";
String SHARED_CACHE_MODE_DEFAULT = "disabled";
/**
* @deprecated Symlinks are always on and cannot be disabled.
*/
@Deprecated
public static final String CACHE_SYMLINK = "mapreduce.job.cache.symlink.create";
public static final String MAPREDUCE_JOB_USER_CLASSPATH_FIRST = "mapreduce.job.user.classpath.first";
public static final String MAPREDUCE_JOB_CLASSLOADER = "mapreduce.job.classloader";
/**
* A comma-separated list of services that function as ShuffleProvider aux-services
* (in addition to the built-in ShuffleHandler).
* These services can serve shuffle requests from reducetasks.
*/
public static final String MAPREDUCE_JOB_SHUFFLE_PROVIDER_SERVICES = "mapreduce.job.shuffle.provider.services";
public static final String MAPREDUCE_JOB_CLASSLOADER_SYSTEM_CLASSES = "mapreduce.job.classloader.system.classes";
public static final String MAPREDUCE_JVM_SYSTEM_PROPERTIES_TO_LOG = "mapreduce.jvm.system-properties-to-log";
public static final String DEFAULT_MAPREDUCE_JVM_SYSTEM_PROPERTIES_TO_LOG =
"os.name,os.version,java.home,java.runtime.version,java.vendor," +
"java.version,java.vm.name,java.class.path,java.io.tmpdir,user.dir,user.name";
public static final String IO_SORT_FACTOR = "mapreduce.task.io.sort.factor";
public static final int DEFAULT_IO_SORT_FACTOR = 10;
public static final String IO_SORT_MB = "mapreduce.task.io.sort.mb";
public static final int DEFAULT_IO_SORT_MB = 100;
public static final String INDEX_CACHE_MEMORY_LIMIT = "mapreduce.task.index.cache.limit.bytes";
public static final String PRESERVE_FAILED_TASK_FILES = "mapreduce.task.files.preserve.failedtasks";
public static final String PRESERVE_FILES_PATTERN = "mapreduce.task.files.preserve.filepattern";
public static final String TASK_DEBUGOUT_LINES = "mapreduce.task.debugout.lines";
public static final String RECORDS_BEFORE_PROGRESS = "mapreduce.task.merge.progress.records";
public static final String SKIP_START_ATTEMPTS = "mapreduce.task.skip.start.attempts";
public static final String TASK_ATTEMPT_ID = "mapreduce.task.attempt.id";
public static final String TASK_ISMAP = "mapreduce.task.ismap";
public static final boolean DEFAULT_TASK_ISMAP = true;
public static final String TASK_PARTITION = "mapreduce.task.partition";
public static final String TASK_PROFILE = "mapreduce.task.profile";
public static final String TASK_PROFILE_PARAMS = "mapreduce.task.profile.params";
public static final String DEFAULT_TASK_PROFILE_PARAMS =
"-agentlib:hprof=cpu=samples,heap=sites,force=n,thread=y,"
+ "verbose=n,file=%s";
public static final String NUM_MAP_PROFILES = "mapreduce.task.profile.maps";
public static final String NUM_REDUCE_PROFILES = "mapreduce.task.profile.reduces";
public static final String TASK_MAP_PROFILE_PARAMS = "mapreduce.task.profile.map.params";
public static final String TASK_REDUCE_PROFILE_PARAMS = "mapreduce.task.profile.reduce.params";
public static final String TASK_TIMEOUT = "mapreduce.task.timeout";
long DEFAULT_TASK_TIMEOUT_MILLIS = 5 * 60 * 1000L;
/**
* The max timeout before receiving remote task's first heartbeat.
* This parameter is in order to avoid waiting for the container
* to start indefinitely, which made task stuck in the NEW state.
*/
String TASK_STUCK_TIMEOUT_MS = "mapreduce.task.stuck.timeout-ms";
long DEFAULT_TASK_STUCK_TIMEOUT_MS = 10 * 60 * 1000L;
String TASK_PROGRESS_REPORT_INTERVAL =
"mapreduce.task.progress-report.interval";
public static final String TASK_TIMEOUT_CHECK_INTERVAL_MS = "mapreduce.task.timeout.check-interval-ms";
public static final String TASK_EXIT_TIMEOUT = "mapreduce.task.exit.timeout";
public static final int TASK_EXIT_TIMEOUT_DEFAULT = 60 * 1000;
public static final String TASK_EXIT_TIMEOUT_CHECK_INTERVAL_MS = "mapreduce.task.exit.timeout.check-interval-ms";
public static final int TASK_EXIT_TIMEOUT_CHECK_INTERVAL_MS_DEFAULT = 20 * 1000;
/**
* TaskAttemptListenerImpl will log the task progress when the delta progress
* is larger than or equal the defined value.
* The double value has to be between 0, and 1 with two decimals.
*/
String TASK_LOG_PROGRESS_DELTA_THRESHOLD =
"mapreduce.task.log.progress.delta.threshold";
/**
* Default delta progress is set to 5%.
*/
double TASK_LOG_PROGRESS_DELTA_THRESHOLD_DEFAULT = 0.05;
/**
* TaskAttemptListenerImpl will log the task progress when the defined value
* in seconds expires.
* This helps to debug task attempts that are doing very slow progress.
*/
String TASK_LOG_PROGRESS_WAIT_INTERVAL_SECONDS =
"mapreduce.task.log.progress.wait.interval-seconds";
/**
* Default period to log the task attempt progress is 60 seconds.
*/
long TASK_LOG_PROGRESS_WAIT_INTERVAL_SECONDS_DEFAULT = 60L;
public static final String TASK_ID = "mapreduce.task.id";
public static final String TASK_OUTPUT_DIR = "mapreduce.task.output.dir";
public static final String TASK_USERLOG_LIMIT = "mapreduce.task.userlog.limit.kb";
public static final String MAP_SORT_SPILL_PERCENT = "mapreduce.map.sort.spill.percent";
public static final String MAP_INPUT_FILE = "mapreduce.map.input.file";
public static final String MAP_INPUT_PATH = "mapreduce.map.input.length";
public static final String MAP_INPUT_START = "mapreduce.map.input.start";
/**
* Configuration key for specifying memory requirement for the mapper.
* Kept for backward-compatibility, mapreduce.map.resource.memory
* is the new preferred way to specify this.
*/
public static final String MAP_MEMORY_MB = "mapreduce.map.memory.mb";
public static final int DEFAULT_MAP_MEMORY_MB = 1024;
/**
* Configuration key for specifying CPU requirement for the mapper.
* Kept for backward-compatibility, mapreduce.map.resource.vcores
* is the new preferred way to specify this.
*/
public static final String MAP_CPU_VCORES = "mapreduce.map.cpu.vcores";
public static final int DEFAULT_MAP_CPU_VCORES = 1;
/**
* Custom resource names required by the mapper should be
* appended to this prefix, the value's format is {amount}[ ][{unit}].
* If no unit is defined, the default unit will be used.
* Standard resource names: memory (default unit: Mi), vcores
*/
public static final String MAP_RESOURCE_TYPE_PREFIX =
"mapreduce.map.resource.";
/**
* Resource type name for CPU vcores.
*/
public static final String RESOURCE_TYPE_NAME_VCORE = "vcores";
/**
* Resource type name for memory.
*/
public static final String RESOURCE_TYPE_NAME_MEMORY = "memory";
/**
* Alternative resource type name for memory.
*/
public static final String RESOURCE_TYPE_ALTERNATIVE_NAME_MEMORY =
"memory-mb";
public static final String MAP_ENV = "mapreduce.map.env";
public static final String MAP_JAVA_OPTS = "mapreduce.map.java.opts";
public static final String MAP_MAX_ATTEMPTS = "mapreduce.map.maxattempts";
public static final String MAP_DEBUG_SCRIPT = "mapreduce.map.debug.script";
public static final String MAP_SPECULATIVE = "mapreduce.map.speculative";
public static final String MAP_FAILURES_MAX_PERCENT = "mapreduce.map.failures.maxpercent";
public static final String MAP_SKIP_INCR_PROC_COUNT = "mapreduce.map.skip.proc-count.auto-incr";
public static final String MAP_SKIP_MAX_RECORDS = "mapreduce.map.skip.maxrecords";
public static final String MAP_COMBINE_MIN_SPILLS = "mapreduce.map.combine.minspills";
public static final String MAP_OUTPUT_COMPRESS = "mapreduce.map.output.compress";
public static final String MAP_OUTPUT_COMPRESS_CODEC = "mapreduce.map.output.compress.codec";
public static final String MAP_OUTPUT_KEY_CLASS = "mapreduce.map.output.key.class";
public static final String MAP_OUTPUT_VALUE_CLASS = "mapreduce.map.output.value.class";
public static final String MAP_OUTPUT_KEY_FIELD_SEPARATOR = "mapreduce.map.output.key.field.separator";
/**
* @deprecated Use {@link #MAP_OUTPUT_KEY_FIELD_SEPARATOR}
*/
@Deprecated
public static final String MAP_OUTPUT_KEY_FIELD_SEPERATOR = MAP_OUTPUT_KEY_FIELD_SEPARATOR;
public static final String MAP_LOG_LEVEL = "mapreduce.map.log.level";
public static final String REDUCE_LOG_LEVEL = "mapreduce.reduce.log.level";
public static final String DEFAULT_LOG_LEVEL = "INFO";
public static final String REDUCE_MERGE_INMEM_THRESHOLD = "mapreduce.reduce.merge.inmem.threshold";
public static final String REDUCE_INPUT_BUFFER_PERCENT = "mapreduce.reduce.input.buffer.percent";
public static final String REDUCE_MARKRESET_BUFFER_PERCENT = "mapreduce.reduce.markreset.buffer.percent";
public static final String REDUCE_MARKRESET_BUFFER_SIZE = "mapreduce.reduce.markreset.buffer.size";
/**
* Configuration key for specifying memory requirement for the reducer.
* Kept for backward-compatibility, mapreduce.reduce.resource.memory
* is the new preferred way to specify this.
*/
public static final String REDUCE_MEMORY_MB = "mapreduce.reduce.memory.mb";
public static final int DEFAULT_REDUCE_MEMORY_MB = 1024;
/**
* Configuration key for specifying CPU requirement for the reducer.
* Kept for backward-compatibility, mapreduce.reduce.resource.vcores
* is the new preferred way to specify this.
*/
public static final String REDUCE_CPU_VCORES = "mapreduce.reduce.cpu.vcores";
public static final int DEFAULT_REDUCE_CPU_VCORES = 1;
/**
* Resource names required by the reducer should be
* appended to this prefix, the value's format is {amount}[ ][{unit}].
* If no unit is defined, the default unit will be used.
* Standard resource names: memory (default unit: Mi), vcores
*/
public static final String REDUCE_RESOURCE_TYPE_PREFIX =
"mapreduce.reduce.resource.";
public static final String REDUCE_MEMORY_TOTAL_BYTES = "mapreduce.reduce.memory.totalbytes";
public static final String SHUFFLE_INPUT_BUFFER_PERCENT = "mapreduce.reduce.shuffle.input.buffer.percent";
public static final float DEFAULT_SHUFFLE_INPUT_BUFFER_PERCENT = 0.70f;
public static final String SHUFFLE_MEMORY_LIMIT_PERCENT
= "mapreduce.reduce.shuffle.memory.limit.percent";
public static final String SHUFFLE_MERGE_PERCENT = "mapreduce.reduce.shuffle.merge.percent";
public static final float DEFAULT_SHUFFLE_MERGE_PERCENT = 0.66f;
public static final String REDUCE_FAILURES_MAXPERCENT = "mapreduce.reduce.failures.maxpercent";
public static final String REDUCE_ENV = "mapreduce.reduce.env";
public static final String REDUCE_JAVA_OPTS = "mapreduce.reduce.java.opts";
public static final String MAPREDUCE_JOB_DIR = "mapreduce.job.dir";
public static final String REDUCE_MAX_ATTEMPTS = "mapreduce.reduce.maxattempts";
public static final String SHUFFLE_PARALLEL_COPIES = "mapreduce.reduce.shuffle.parallelcopies";
public static final String REDUCE_DEBUG_SCRIPT = "mapreduce.reduce.debug.script";
public static final String REDUCE_SPECULATIVE = "mapreduce.reduce.speculative";
public static final String SHUFFLE_CONNECT_TIMEOUT = "mapreduce.reduce.shuffle.connect.timeout";
public static final String SHUFFLE_READ_TIMEOUT = "mapreduce.reduce.shuffle.read.timeout";
public static final String SHUFFLE_FETCH_FAILURES = "mapreduce.reduce.shuffle.maxfetchfailures";
public static final String MAX_ALLOWED_FETCH_FAILURES_FRACTION = "mapreduce.reduce.shuffle.max-fetch-failures-fraction";
public static final float DEFAULT_MAX_ALLOWED_FETCH_FAILURES_FRACTION = 0.5f;
public static final String MAX_FETCH_FAILURES_NOTIFICATIONS = "mapreduce.reduce.shuffle.max-fetch-failures-notifications";
public static final int DEFAULT_MAX_FETCH_FAILURES_NOTIFICATIONS = 3;
public static final String SHUFFLE_FETCH_RETRY_INTERVAL_MS = "mapreduce.reduce.shuffle.fetch.retry.interval-ms";
/** Default interval that fetcher retry to fetch during NM restart.*/
public final static int DEFAULT_SHUFFLE_FETCH_RETRY_INTERVAL_MS = 1000;
public static final String SHUFFLE_FETCH_RETRY_TIMEOUT_MS = "mapreduce.reduce.shuffle.fetch.retry.timeout-ms";
public static final String SHUFFLE_FETCH_RETRY_ENABLED = "mapreduce.reduce.shuffle.fetch.retry.enabled";
public static final String SHUFFLE_NOTIFY_READERROR = "mapreduce.reduce.shuffle.notify.readerror";
public static final String MAX_SHUFFLE_FETCH_RETRY_DELAY = "mapreduce.reduce.shuffle.retry-delay.max.ms";
public static final long DEFAULT_MAX_SHUFFLE_FETCH_RETRY_DELAY = 60000;
public static final String MAX_SHUFFLE_FETCH_HOST_FAILURES = "mapreduce.reduce.shuffle.max-host-failures";
public static final int DEFAULT_MAX_SHUFFLE_FETCH_HOST_FAILURES = 5;
public static final String REDUCE_SKIP_INCR_PROC_COUNT = "mapreduce.reduce.skip.proc-count.auto-incr";
public static final String REDUCE_SKIP_MAXGROUPS = "mapreduce.reduce.skip.maxgroups";
public static final String REDUCE_MEMTOMEM_THRESHOLD = "mapreduce.reduce.merge.memtomem.threshold";
public static final String REDUCE_MEMTOMEM_ENABLED = "mapreduce.reduce.merge.memtomem.enabled";
public static final String COMBINE_RECORDS_BEFORE_PROGRESS = "mapreduce.task.combine.progress.records";
public static final String JOB_NAMENODES = "mapreduce.job.hdfs-servers";
public static final String JOB_NAMENODES_TOKEN_RENEWAL_EXCLUDE = "mapreduce.job.hdfs-servers.token-renewal.exclude";
public static final String JOB_JOBTRACKER_ID = "mapreduce.job.kerberos.jtprinicipal";
public static final String JOB_CANCEL_DELEGATION_TOKEN = "mapreduce.job.complete.cancel.delegation.tokens";
public static final String JOB_ACL_VIEW_JOB = "mapreduce.job.acl-view-job";
public static final String DEFAULT_JOB_ACL_VIEW_JOB = " ";
public static final String JOB_ACL_MODIFY_JOB = "mapreduce.job.acl-modify-job";
public static final String DEFAULT_JOB_ACL_MODIFY_JOB = " ";
public static final String JOB_RUNNING_MAP_LIMIT =
"mapreduce.job.running.map.limit";
public static final int DEFAULT_JOB_RUNNING_MAP_LIMIT = 0;
public static final String JOB_RUNNING_REDUCE_LIMIT =
"mapreduce.job.running.reduce.limit";
public static final int DEFAULT_JOB_RUNNING_REDUCE_LIMIT = 0;
/* Config for Limit on the number of map tasks allowed per job
* There is no limit if this value is negative.
*/
public static final String JOB_MAX_MAP =
"mapreduce.job.max.map";
public static final int DEFAULT_JOB_MAX_MAP = -1;
/* config for tracking the local file where all the credentials for the job
* credentials.
*/
public static final String MAPREDUCE_JOB_CREDENTIALS_BINARY =
"mapreduce.job.credentials.binary";
/* Configs for tracking ids of tokens used by a job */
public static final String JOB_TOKEN_TRACKING_IDS_ENABLED =
"mapreduce.job.token.tracking.ids.enabled";
public static final boolean DEFAULT_JOB_TOKEN_TRACKING_IDS_ENABLED = false;
public static final String JOB_TOKEN_TRACKING_IDS =
"mapreduce.job.token.tracking.ids";
public static final String JOB_SUBMITHOST =
"mapreduce.job.submithostname";
public static final String JOB_SUBMITHOSTADDR =
"mapreduce.job.submithostaddress";
public static final String COUNTERS_MAX_KEY = "mapreduce.job.counters.max";
public static final int COUNTERS_MAX_DEFAULT = 120;
public static final String COUNTER_GROUP_NAME_MAX_KEY = "mapreduce.job.counters.group.name.max";
public static final int COUNTER_GROUP_NAME_MAX_DEFAULT = 128;
public static final String COUNTER_NAME_MAX_KEY = "mapreduce.job.counters.counter.name.max";
public static final int COUNTER_NAME_MAX_DEFAULT = 64;
public static final String COUNTER_GROUPS_MAX_KEY = "mapreduce.job.counters.groups.max";
public static final int COUNTER_GROUPS_MAX_DEFAULT = 50;
public static final String JOB_UBERTASK_ENABLE =
"mapreduce.job.ubertask.enable";
public static final String JOB_UBERTASK_MAXMAPS =
"mapreduce.job.ubertask.maxmaps";
public static final String JOB_UBERTASK_MAXREDUCES =
"mapreduce.job.ubertask.maxreduces";
public static final String JOB_UBERTASK_MAXBYTES =
"mapreduce.job.ubertask.maxbytes";
public static final String MAPREDUCE_JOB_EMIT_TIMELINE_DATA =
"mapreduce.job.emit-timeline-data";
public static final boolean DEFAULT_MAPREDUCE_JOB_EMIT_TIMELINE_DATA =
false;
public static final String MR_PREFIX = "yarn.app.mapreduce.";
public static final String MR_AM_PREFIX = MR_PREFIX + "am.";
/** The number of client retries to the AM - before reconnecting to the RM
* to fetch Application State.
*/
public static final String MR_CLIENT_TO_AM_IPC_MAX_RETRIES =
MR_PREFIX + "client-am.ipc.max-retries";
public static final int DEFAULT_MR_CLIENT_TO_AM_IPC_MAX_RETRIES = 3;
/** The number of client retries on socket timeouts to the AM - before
* reconnecting to the RM to fetch Application Status.
*/
public static final String MR_CLIENT_TO_AM_IPC_MAX_RETRIES_ON_TIMEOUTS =
MR_PREFIX + "client-am.ipc.max-retries-on-timeouts";
public static final int
DEFAULT_MR_CLIENT_TO_AM_IPC_MAX_RETRIES_ON_TIMEOUTS = 3;
/**
* The number of client retries to the RM/HS before throwing exception.
*/
public static final String MR_CLIENT_MAX_RETRIES =
MR_PREFIX + "client.max-retries";
public static final int DEFAULT_MR_CLIENT_MAX_RETRIES = 3;
/**
* How many times to retry jobclient calls (via getjob)
*/
public static final String MR_CLIENT_JOB_MAX_RETRIES =
MR_PREFIX + "client.job.max-retries";
public static final int DEFAULT_MR_CLIENT_JOB_MAX_RETRIES = 3;
/**
* How long to wait between jobclient retries on failure
*/
public static final String MR_CLIENT_JOB_RETRY_INTERVAL =
MR_PREFIX + "client.job.retry-interval";
public static final long DEFAULT_MR_CLIENT_JOB_RETRY_INTERVAL =
2000;
/** The staging directory for map reduce.*/
public static final String MR_AM_STAGING_DIR =
MR_AM_PREFIX+"staging-dir";
public static final String DEFAULT_MR_AM_STAGING_DIR =
"/tmp/hadoop-yarn/staging";
/** The amount of memory the MR app master needs.
* Kept for backward-compatibility, yarn.app.mapreduce.am.resource.memory is
* the new preferred way to specify this
*/
public static final String MR_AM_VMEM_MB =
MR_AM_PREFIX+"resource.mb";
public static final int DEFAULT_MR_AM_VMEM_MB = 1536;
/** The number of virtual cores the MR app master needs.*/
public static final String MR_AM_CPU_VCORES =
MR_AM_PREFIX+"resource.cpu-vcores";
public static final int DEFAULT_MR_AM_CPU_VCORES = 1;
/**
* Resource names required by the MR AM should be
* appended to this prefix, the value's format is {amount}[ ][{unit}].
* If no unit is defined, the default unit will be used
* Standard resource names: memory (default unit: Mi), vcores
*/
public static final String MR_AM_RESOURCE_PREFIX =
MR_AM_PREFIX + "resource.";
/** Command line arguments passed to the MR app master.*/
public static final String MR_AM_COMMAND_OPTS =
MR_AM_PREFIX+"command-opts";
public static final String DEFAULT_MR_AM_COMMAND_OPTS = "-Xmx1024m";
/** Admin command opts passed to the MR app master.*/
public static final String MR_AM_ADMIN_COMMAND_OPTS =
MR_AM_PREFIX+"admin-command-opts";
public static final String DEFAULT_MR_AM_ADMIN_COMMAND_OPTS = "";
/** Root Logging level passed to the MR app master.*/
public static final String MR_AM_LOG_LEVEL =
MR_AM_PREFIX+"log.level";
public static final String DEFAULT_MR_AM_LOG_LEVEL = "INFO";
public static final String MR_AM_LOG_KB =
MR_AM_PREFIX + "container.log.limit.kb";
public static final int DEFAULT_MR_AM_LOG_KB = 0; // don't roll
public static final String MR_AM_LOG_BACKUPS =
MR_AM_PREFIX + "container.log.backups";
public static final int DEFAULT_MR_AM_LOG_BACKUPS = 0;
/**The number of splits when reporting progress in MR*/
public static final String MR_AM_NUM_PROGRESS_SPLITS =
MR_AM_PREFIX+"num-progress-splits";
public static final int DEFAULT_MR_AM_NUM_PROGRESS_SPLITS = 12;
/**
* Upper limit on the number of threads user to launch containers in the app
* master. Expect level config, you shouldn't be needing it in most cases.
*/
public static final String MR_AM_CONTAINERLAUNCHER_THREAD_COUNT_LIMIT =
MR_AM_PREFIX+"containerlauncher.thread-count-limit";
public static final int DEFAULT_MR_AM_CONTAINERLAUNCHER_THREAD_COUNT_LIMIT =
500;
/**
* The initial size of thread pool to launch containers in the app master
*/
public static final String MR_AM_CONTAINERLAUNCHER_THREADPOOL_INITIAL_SIZE =
MR_AM_PREFIX+"containerlauncher.threadpool-initial-size";
public static final int DEFAULT_MR_AM_CONTAINERLAUNCHER_THREADPOOL_INITIAL_SIZE =
10;
/** Number of threads to handle job client RPC requests.*/
public static final String MR_AM_JOB_CLIENT_THREAD_COUNT =
MR_AM_PREFIX + "job.client.thread-count";
public static final int DEFAULT_MR_AM_JOB_CLIENT_THREAD_COUNT = 1;
/**
* Range of ports that the MapReduce AM can use when binding. Leave blank
* if you want all possible ports.
*/
public static final String MR_AM_JOB_CLIENT_PORT_RANGE =
MR_AM_PREFIX + "job.client.port-range";
/**
* Range of ports that the MapReduce AM can use when binding for its webapp.
* Leave blank if you want all possible ports.
*/
String MR_AM_WEBAPP_PORT_RANGE = MR_AM_PREFIX + "webapp.port-range";
/**
* True if the MR AM should use HTTPS for its webapp. If
* {@link org.apache.hadoop.yarn.conf.YarnConfiguration#RM_APPLICATION_HTTPS_POLICY}
* is set to LENIENT or STRICT, the MR AM will automatically use the
* keystore provided by YARN with a certificate for the MR AM webapp, unless
* provided by the user.
*/
String MR_AM_WEBAPP_HTTPS_ENABLED = MR_AM_PREFIX + "webapp.https.enabled";
boolean DEFAULT_MR_AM_WEBAPP_HTTPS_ENABLED = false;
/**
* True if the MR AM webapp should require client HTTPS authentication (i.e.
* the proxy server (RM) should present a certificate to the MR AM webapp).
* If {@link org.apache.hadoop.yarn.conf.YarnConfiguration#RM_APPLICATION_HTTPS_POLICY}
* is set to LENIENT or STRICT, the MR AM will automatically use the
* truststore provided by YARN with the RMs certificate, unless provided by
* the user.
*/
String MR_AM_WEBAPP_HTTPS_CLIENT_AUTH =
MR_AM_PREFIX + "webapp.https.client.auth";
boolean DEFAULT_MR_AM_WEBAPP_HTTPS_CLIENT_AUTH = false;
/** Enable blacklisting of nodes in the job.*/
public static final String MR_AM_JOB_NODE_BLACKLISTING_ENABLE =
MR_AM_PREFIX + "job.node-blacklisting.enable";
/** Ignore blacklisting if a certain percentage of nodes have been blacklisted */
public static final String MR_AM_IGNORE_BLACKLISTING_BLACKLISTED_NODE_PERECENT =
MR_AM_PREFIX + "job.node-blacklisting.ignore-threshold-node-percent";
public static final int DEFAULT_MR_AM_IGNORE_BLACKLISTING_BLACKLISTED_NODE_PERCENT =
33;
/** Enable job recovery.*/
public static final String MR_AM_JOB_RECOVERY_ENABLE =
MR_AM_PREFIX + "job.recovery.enable";
public static final boolean MR_AM_JOB_RECOVERY_ENABLE_DEFAULT = true;
/**
* Limit on the number of reducers that can be preempted to ensure that at
* least one map task can run if it needs to. Percentage between 0.0 and 1.0
*/
public static final String MR_AM_JOB_REDUCE_PREEMPTION_LIMIT =
MR_AM_PREFIX + "job.reduce.preemption.limit";
public static final float DEFAULT_MR_AM_JOB_REDUCE_PREEMPTION_LIMIT = 0.5f;
/**
* Policy class encoding responses to preemption requests.
*/
public static final String MR_AM_PREEMPTION_POLICY =
MR_AM_PREFIX + "preemption.policy";
/** AM ACL disabled. **/
public static final String JOB_AM_ACCESS_DISABLED =
"mapreduce.job.am-access-disabled";
public static final boolean DEFAULT_JOB_AM_ACCESS_DISABLED = false;
/**
* Limit reduces starting until a certain percentage of maps have finished.
* Percentage between 0.0 and 1.0
*/
public static final String MR_AM_JOB_REDUCE_RAMPUP_UP_LIMIT =
MR_AM_PREFIX + "job.reduce.rampup.limit";
public static final float DEFAULT_MR_AM_JOB_REDUCE_RAMP_UP_LIMIT = 0.5f;
/** The class that should be used for speculative execution calculations.*/
public static final String MR_AM_JOB_SPECULATOR =
MR_AM_PREFIX + "job.speculator.class";
/** Class used to estimate task resource needs.*/
public static final String MR_AM_TASK_ESTIMATOR =
MR_AM_PREFIX + "job.task.estimator.class";
/** The lambda value in the smoothing function of the task estimator.*/
public static final String MR_AM_TASK_ESTIMATOR_SMOOTH_LAMBDA_MS =
MR_AM_PREFIX
+ "job.task.estimator.exponential.smooth.lambda-ms";
public static final long DEFAULT_MR_AM_TASK_ESTIMATOR_SMOOTH_LAMBDA_MS =
1000L * 60;
/** true if the smoothing rate should be exponential.*/
public static final String MR_AM_TASK_ESTIMATOR_EXPONENTIAL_RATE_ENABLE =
MR_AM_PREFIX + "job.task.estimator.exponential.smooth.rate";
/** The lambda value in the smoothing function of the task estimator.*/
String MR_AM_TASK_ESTIMATOR_SIMPLE_SMOOTH_LAMBDA_MS =
MR_AM_PREFIX
+ "job.task.estimator.simple.exponential.smooth.lambda-ms";
long DEFAULT_MR_AM_TASK_ESTIMATOR_SIMPLE_SMOOTH_LAMBDA_MS = 1000L * 120;
/**
* The window length in the simple exponential smoothing that considers the
* task attempt is stagnated.
*/
String MR_AM_TASK_ESTIMATOR_SIMPLE_SMOOTH_STAGNATED_MS =
MR_AM_PREFIX
+ "job.task.estimator.simple.exponential.smooth.stagnated-ms";
long DEFAULT_MR_AM_TASK_ESTIMATOR_SIMPLE_SMOOTH_STAGNATED_MS =
1000L * 360;
/**
* The number of initial readings that the estimator ignores before giving a
* prediction. At the beginning the smooth estimator won't be accurate in
* prediction.
*/
String MR_AM_TASK_ESTIMATOR_SIMPLE_SMOOTH_SKIP_INITIALS =
MR_AM_PREFIX
+ "job.task.estimator.simple.exponential.smooth.skip-initials";
/**
* The default number of reading the estimators is going to ignore before
* returning the smooth exponential prediction.
*/
int DEFAULT_MR_AM_TASK_ESTIMATOR_SIMPLE_SMOOTH_INITIALS = 24;
/** The number of threads used to handle task RPC calls.*/
public static final String MR_AM_TASK_LISTENER_THREAD_COUNT =
MR_AM_PREFIX + "job.task.listener.thread-count";
public static final int DEFAULT_MR_AM_TASK_LISTENER_THREAD_COUNT = 30;
/** How often the AM should send heartbeats to the RM.*/
public static final String MR_AM_TO_RM_HEARTBEAT_INTERVAL_MS =
MR_AM_PREFIX + "scheduler.heartbeat.interval-ms";
public static final int DEFAULT_MR_AM_TO_RM_HEARTBEAT_INTERVAL_MS = 1000;
/** Whether to consider ping from tasks in liveliness check. */
String MR_TASK_ENABLE_PING_FOR_LIVELINESS_CHECK =
"mapreduce.task.ping-for-liveliness-check.enabled";
boolean DEFAULT_MR_TASK_ENABLE_PING_FOR_LIVELINESS_CHECK
= false;
/**
* If contact with RM is lost, the AM will wait MR_AM_TO_RM_WAIT_INTERVAL_MS
* milliseconds before aborting. During this interval, AM will still try
* to contact the RM.
*/
public static final String MR_AM_TO_RM_WAIT_INTERVAL_MS =
MR_AM_PREFIX + "scheduler.connection.wait.interval-ms";
public static final int DEFAULT_MR_AM_TO_RM_WAIT_INTERVAL_MS = 360000;
/**
* How long to wait in milliseconds for the output committer to cancel
* an operation when the job is being killed
*/
public static final String MR_AM_COMMITTER_CANCEL_TIMEOUT_MS =
MR_AM_PREFIX + "job.committer.cancel-timeout";
public static final int DEFAULT_MR_AM_COMMITTER_CANCEL_TIMEOUT_MS =
60 * 1000;
/**
* Defines a time window in milliseconds for output committer operations.
* If contact with the RM has occurred within this window then commit
* operations are allowed, otherwise the AM will not allow output committer
* operations until contact with the RM has been re-established.
*/
public static final String MR_AM_COMMIT_WINDOW_MS =
MR_AM_PREFIX + "job.committer.commit-window";
public static final int DEFAULT_MR_AM_COMMIT_WINDOW_MS = 10 * 1000;
/**
* Boolean. Create the base dirs in the JobHistoryEventHandler
* Set to false for multi-user clusters. This is an internal config that
* is set by the MR framework and read by it too.
*/
public static final String MR_AM_CREATE_JH_INTERMEDIATE_BASE_DIR =
MR_AM_PREFIX + "create-intermediate-jh-base-dir";
public static final String MR_AM_HISTORY_MAX_UNFLUSHED_COMPLETE_EVENTS =
MR_AM_PREFIX + "history.max-unflushed-events";
public static final int DEFAULT_MR_AM_HISTORY_MAX_UNFLUSHED_COMPLETE_EVENTS =
200;
public static final String MR_AM_HISTORY_JOB_COMPLETE_UNFLUSHED_MULTIPLIER =
MR_AM_PREFIX + "history.job-complete-unflushed-multiplier";
public static final int DEFAULT_MR_AM_HISTORY_JOB_COMPLETE_UNFLUSHED_MULTIPLIER =
30;
public static final String MR_AM_HISTORY_COMPLETE_EVENT_FLUSH_TIMEOUT_MS =
MR_AM_PREFIX + "history.complete-event-flush-timeout";
public static final long DEFAULT_MR_AM_HISTORY_COMPLETE_EVENT_FLUSH_TIMEOUT_MS =
30 * 1000l;
public static final String MR_AM_HISTORY_USE_BATCHED_FLUSH_QUEUE_SIZE_THRESHOLD =
MR_AM_PREFIX + "history.use-batched-flush.queue-size.threshold";
public static final int DEFAULT_MR_AM_HISTORY_USE_BATCHED_FLUSH_QUEUE_SIZE_THRESHOLD =
50;
public static final String MR_AM_HARD_KILL_TIMEOUT_MS =
MR_AM_PREFIX + "hard-kill-timeout-ms";
public static final long DEFAULT_MR_AM_HARD_KILL_TIMEOUT_MS =
10 * 1000l;
/**
* Duration to wait before forcibly preempting a reducer to allow
* allocating new mappers, even when YARN reports positive headroom.
*/
public static final String MR_JOB_REDUCER_UNCONDITIONAL_PREEMPT_DELAY_SEC =
"mapreduce.job.reducer.unconditional-preempt.delay.sec";
public static final int
DEFAULT_MR_JOB_REDUCER_UNCONDITIONAL_PREEMPT_DELAY_SEC = 5 * 60;
/**
* Duration to wait before preempting a reducer, when there is no headroom
* to allocate new mappers.
*/
public static final String MR_JOB_REDUCER_PREEMPT_DELAY_SEC =
"mapreduce.job.reducer.preempt.delay.sec";
public static final int DEFAULT_MR_JOB_REDUCER_PREEMPT_DELAY_SEC = 0;
public static final String MR_AM_ENV =
MR_AM_PREFIX + "env";
public static final String MR_AM_ADMIN_USER_ENV =
MR_AM_PREFIX + "admin.user.env";
// although the AM admin user env default should be the same as the task user
// env default, there are problems in making it work on Windows currently
// MAPREDUCE-6588 should address the issue and set it to a proper non-empty
// value
public static final String DEFAULT_MR_AM_ADMIN_USER_ENV =
Shell.WINDOWS ?
"" :
"LD_LIBRARY_PATH=" + Apps.crossPlatformify("HADOOP_COMMON_HOME") +
"/lib/native";
public static final String MR_AM_PROFILE = MR_AM_PREFIX + "profile";
public static final boolean DEFAULT_MR_AM_PROFILE = false;
public static final String MR_AM_PROFILE_PARAMS = MR_AM_PREFIX
+ "profile.params";
public static final String MAPRED_MAP_ADMIN_JAVA_OPTS =
"mapreduce.admin.map.child.java.opts";
public static final String MAPRED_REDUCE_ADMIN_JAVA_OPTS =
"mapreduce.admin.reduce.child.java.opts";
public static final String DEFAULT_MAPRED_ADMIN_JAVA_OPTS =
"-Djava.net.preferIPv4Stack=true " +
"-Dhadoop.metrics.log.level=WARN ";
public static final String MAPRED_ADMIN_USER_SHELL =
"mapreduce.admin.user.shell";
public static final String DEFAULT_SHELL = "/bin/bash";
public static final String MAPRED_ADMIN_USER_ENV =
"mapreduce.admin.user.env";
// the "%...%" macros can be expanded prematurely and are probably not OK
// this should be addressed by MAPREDUCE-6588
public static final String DEFAULT_MAPRED_ADMIN_USER_ENV =
Shell.WINDOWS ?
"PATH=%PATH%;%HADOOP_COMMON_HOME%\\bin" :
"LD_LIBRARY_PATH=" + Apps.crossPlatformify("HADOOP_COMMON_HOME") +
"/lib/native";
public static final String WORKDIR = "work";
public static final String OUTPUT = "output";
public static final String HADOOP_WORK_DIR = "HADOOP_WORK_DIR";
// Environment variables used by Pipes. (TODO: these
// do not appear to be used by current pipes source code!)
public static final String STDOUT_LOGFILE_ENV = "STDOUT_LOGFILE_ENV";
public static final String STDERR_LOGFILE_ENV = "STDERR_LOGFILE_ENV";
// This should be the directory where splits file gets localized on the node
// running ApplicationMaster.
public static final String JOB_SUBMIT_DIR = "jobSubmitDir";
// This should be the name of the localized job-configuration file on the node
// running ApplicationMaster and Task
public static final String JOB_CONF_FILE = "job.xml";
// This should be the name of the localized job-jar file on the node running
// individual containers/tasks.
public static final String JOB_JAR = "job.jar";
public static final String JOB_SPLIT = "job.split";
public static final String JOB_SPLIT_METAINFO = "job.splitmetainfo";
public static final String APPLICATION_MASTER_CLASS =
"org.apache.hadoop.mapreduce.v2.app.MRAppMaster";
public static final String MAPREDUCE_V2_CHILD_CLASS =
"org.apache.hadoop.mapred.YarnChild";
public static final String APPLICATION_ATTEMPT_ID =
"mapreduce.job.application.attempt.id";
/**
* Job end notification.
*/
public static final String MR_JOB_END_NOTIFICATION_URL =
"mapreduce.job.end-notification.url";
public static final String MR_JOB_END_NOTIFICATION_PROXY =
"mapreduce.job.end-notification.proxy";
public static final String MR_JOB_END_NOTIFICATION_TIMEOUT =
"mapreduce.job.end-notification.timeout";
public static final String MR_JOB_END_RETRY_ATTEMPTS =
"mapreduce.job.end-notification.retry.attempts";
public static final String MR_JOB_END_RETRY_INTERVAL =
"mapreduce.job.end-notification.retry.interval";
public static final String MR_JOB_END_NOTIFICATION_MAX_ATTEMPTS =
"mapreduce.job.end-notification.max.attempts";
public static final String MR_JOB_END_NOTIFICATION_MAX_RETRY_INTERVAL =
"mapreduce.job.end-notification.max.retry.interval";
String MR_JOB_END_NOTIFICATION_CUSTOM_NOTIFIER_CLASS =
"mapreduce.job.end-notification.custom-notifier-class";
public static final int DEFAULT_MR_JOB_END_NOTIFICATION_TIMEOUT =
5000;
/*
* MR AM Service Authorization
*/
public static final String
MR_AM_SECURITY_SERVICE_AUTHORIZATION_TASK_UMBILICAL =
"security.job.task.protocol.acl";
public static final String
MR_AM_SECURITY_SERVICE_AUTHORIZATION_CLIENT =
"security.job.client.protocol.acl";
/**
* CLASSPATH for all YARN MapReduce applications.
*/
public static final String MAPREDUCE_APPLICATION_CLASSPATH =
"mapreduce.application.classpath";
public static final String MAPREDUCE_JOB_LOG4J_PROPERTIES_FILE =
"mapreduce.job.log4j-properties-file";
/**
* Path to MapReduce framework archive
*/
public static final String MAPREDUCE_APPLICATION_FRAMEWORK_PATH =
"mapreduce.application.framework.path";
/**
* Default CLASSPATH for all YARN MapReduce applications constructed with
* platform-agnostic syntax.
*/
@Public
@Unstable
public final String DEFAULT_MAPREDUCE_CROSS_PLATFORM_APPLICATION_CLASSPATH = Apps
.crossPlatformify("HADOOP_MAPRED_HOME")
+ "/share/hadoop/mapreduce/*,"
+ Apps.crossPlatformify("HADOOP_MAPRED_HOME")
+ "/share/hadoop/mapreduce/lib/*";
/**
* Default platform-specific CLASSPATH for all YARN MapReduce applications
* constructed based on client OS syntax.
* <p>
* Note: Use {@link DEFAULT_MAPREDUCE_CROSS_PLATFORM_APPLICATION_CLASSPATH}
* for cross-platform practice i.e. submit an application from a Windows
* client to a Linux/Unix server or vice versa.
* </p>
*/
public final String DEFAULT_MAPREDUCE_APPLICATION_CLASSPATH =
Shell.WINDOWS ? "%HADOOP_MAPRED_HOME%\\share\\hadoop\\mapreduce\\*,"
+ "%HADOOP_MAPRED_HOME%\\share\\hadoop\\mapreduce\\lib\\*"
: "$HADOOP_MAPRED_HOME/share/hadoop/mapreduce/*,"
+ "$HADOOP_MAPRED_HOME/share/hadoop/mapreduce/lib/*";
public static final String WORKFLOW_ID = "mapreduce.workflow.id";
public static final String TASK_LOG_BACKUPS =
MR_PREFIX + "task.container.log.backups";
public static final int DEFAULT_TASK_LOG_BACKUPS = 0; // don't roll
public static final String REDUCE_SEPARATE_SHUFFLE_LOG =
MR_PREFIX + "shuffle.log.separate";
public static final boolean DEFAULT_REDUCE_SEPARATE_SHUFFLE_LOG = true;
public static final String SHUFFLE_LOG_BACKUPS =
MR_PREFIX + "shuffle.log.backups";
public static final int DEFAULT_SHUFFLE_LOG_BACKUPS = 0; // don't roll
public static final String SHUFFLE_LOG_KB =
MR_PREFIX + "shuffle.log.limit.kb";
public static final long DEFAULT_SHUFFLE_LOG_KB = 0L;
public static final String WORKFLOW_NAME = "mapreduce.workflow.name";
public static final String WORKFLOW_NODE_NAME =
"mapreduce.workflow.node.name";
public static final String WORKFLOW_ADJACENCY_PREFIX_STRING =
"mapreduce.workflow.adjacency.";
public static final String WORKFLOW_ADJACENCY_PREFIX_PATTERN =
"^mapreduce\\.workflow\\.adjacency\\..+";
public static final String WORKFLOW_TAGS = "mapreduce.workflow.tags";
/**
* The maximum number of application attempts.
* It is a application-specific setting.
*/
public static final String MR_AM_MAX_ATTEMPTS = "mapreduce.am.max-attempts";
public static final int DEFAULT_MR_AM_MAX_ATTEMPTS = 2;
public static final String MR_APPLICATION_TYPE = "MAPREDUCE";
public static final String TASK_PREEMPTION =
"mapreduce.job.preemption";
public static final String HEAP_MEMORY_MB_RATIO =
"mapreduce.job.heap.memory-mb.ratio";
public static final float DEFAULT_HEAP_MEMORY_MB_RATIO = 0.8f;
public static final String MR_ENCRYPTED_INTERMEDIATE_DATA =
"mapreduce.job.encrypted-intermediate-data";
public static final boolean DEFAULT_MR_ENCRYPTED_INTERMEDIATE_DATA = false;
public static final String MR_ENCRYPTED_INTERMEDIATE_DATA_KEY_SIZE_BITS =
"mapreduce.job.encrypted-intermediate-data-key-size-bits";
public static final int DEFAULT_MR_ENCRYPTED_INTERMEDIATE_DATA_KEY_SIZE_BITS =
128;
public static final String MR_ENCRYPTED_INTERMEDIATE_DATA_BUFFER_KB =
"mapreduce.job.encrypted-intermediate-data.buffer.kb";
public static final int DEFAULT_MR_ENCRYPTED_INTERMEDIATE_DATA_BUFFER_KB =
128;
/**
* The maximum number of resources a map reduce job is allowed to submit for
* localization via files, libjars, archives, and jobjar command line
* arguments and through the distributed cache. If set to 0 the limit is
* ignored.
*/
String MAX_RESOURCES = "mapreduce.job.cache.limit.max-resources";
int MAX_RESOURCES_DEFAULT = 0;
/**
* The maximum size (in MB) a map reduce job is allowed to submit for
* localization via files, libjars, archives, and jobjar command line
* arguments and through the distributed cache. If set to 0 the limit is
* ignored.
*/
String MAX_RESOURCES_MB = "mapreduce.job.cache.limit.max-resources-mb";
long MAX_RESOURCES_MB_DEFAULT = 0;
/**
* The maximum size (in MB) of a single resource a map reduce job is allow to
* submit for localization via files, libjars, archives, and jobjar command
* line arguments and through the distributed cache. If set to 0 the limit is
* ignored.
*/
String MAX_SINGLE_RESOURCE_MB =
"mapreduce.job.cache.limit.max-single-resource-mb";
long MAX_SINGLE_RESOURCE_MB_DEFAULT = 0;
/**
* Number of OPPORTUNISTIC Containers per 100 containers that will be
* requested by the MRAppMaster. The Default value is 0, which implies all
* maps will be guaranteed. A value of 100 means all maps will be requested
* as opportunistic. For any other value say 'x', the FIRST 'x' maps
* requested by the AM will be opportunistic. If the total number of maps
* for the job is less than 'x', then ALL maps will be OPPORTUNISTIC
*/
public static final String MR_NUM_OPPORTUNISTIC_MAPS_PERCENT =
"mapreduce.job.num-opportunistic-maps-percent";
public static final int DEFAULT_MR_NUM_OPPORTUNISTIC_MAPS_PERCENT = 0;
/**
* A comma-separated list of properties whose value will be redacted.
*/
String MR_JOB_REDACTED_PROPERTIES = "mapreduce.job.redacted-properties";
String MR_JOB_SEND_TOKEN_CONF = "mapreduce.job.send-token-conf";
String FINISH_JOB_WHEN_REDUCERS_DONE =
"mapreduce.job.finish-when-all-reducers-done";
boolean DEFAULT_FINISH_JOB_WHEN_REDUCERS_DONE = true;
String MR_AM_STAGING_DIR_ERASURECODING_ENABLED =
MR_AM_STAGING_DIR + ".erasurecoding.enabled";
boolean DEFAULT_MR_AM_STAGING_ERASURECODING_ENABLED = false;
/**
* Prefix for options which are passed in to the filesystem
* after converting the subsequent dotted element to the schema.
*/
@Unstable
String INPUT_FILE_OPTION_PREFIX = "mapreduce.job.input.file.option.";
/**
* Prefix for mandatory options which are passed in to the filesystem
* after converting the subsequent dotted element to the schema.
*/
@Unstable
String INPUT_FILE_MANDATORY_PREFIX = "mapreduce.job.input.file.must.";
String SHUFFLE_KEY_LENGTH = "mapreduce.shuffle-key-length";
int DEFAULT_SHUFFLE_KEY_LENGTH = 64;
}
相关信息
相关文章
0
赞
热门推荐
-
2、 - 优质文章
-
3、 gate.io
-
8、 golang
-
9、 openharmony
-
10、 Vue中input框自动聚焦