hadoop AWSCredentialProviderList 源码

haddop AWSCredentialProviderList 代码

文件路径：/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/AWSCredentialProviderList.java

/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package org.apache.hadoop.fs.s3a;

import java.io.Closeable;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.List;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.stream.Collectors;

import com.amazonaws.AmazonClientException;
import com.amazonaws.auth.AWSCredentials;
import com.amazonaws.auth.AWSCredentialsProvider;
import com.amazonaws.auth.AnonymousAWSCredentials;
import org.apache.hadoop.classification.VisibleForTesting;
import org.apache.hadoop.util.Preconditions;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.fs.s3a.auth.NoAuthWithAWSException;
import org.apache.hadoop.fs.s3a.auth.NoAwsCredentialsException;
import org.apache.hadoop.io.IOUtils;

/**
 * A list of providers.
 *
 * This is similar to the AWS SDK {@code AWSCredentialsProviderChain},
 * except that:
 * <ol>
 *   <li>Allows extra providers to be added dynamically.</li>
 *   <li>If any provider in the chain throws an exception other than
 *   an {@link AmazonClientException}, that is rethrown, rather than
 *   swallowed.</li>
 *   <li>Has some more diagnostics.</li>
 *   <li>On failure, the last "relevant" AmazonClientException raised is
 *   rethrown; exceptions other than 'no credentials' have priority.</li>
 *   <li>Special handling of {@link AnonymousAWSCredentials}.</li>
 * </ol>
 */
@InterfaceAudience.Private
@InterfaceStability.Evolving
public final class AWSCredentialProviderList implements AWSCredentialsProvider,
    AutoCloseable {

  private static final Logger LOG = LoggerFactory.getLogger(
      AWSCredentialProviderList.class);
  public static final String NO_AWS_CREDENTIAL_PROVIDERS
      = "No AWS Credential Providers";

  static final String
      CREDENTIALS_REQUESTED_WHEN_CLOSED
      = "Credentials requested after provider list was closed";

  private final List<AWSCredentialsProvider> providers = new ArrayList<>(1);
  private boolean reuseLastProvider = true;
  private AWSCredentialsProvider lastProvider;

  private final AtomicInteger refCount = new AtomicInteger(1);

  private final AtomicBoolean closed = new AtomicBoolean(false);

  /**
   * The name, which is empty by default.
   * Uses in the code assume if non empty there's a trailing space.
   */
  private String name = "";

  /**
   * Empty instance. This is not ready to be used.
   */
  public AWSCredentialProviderList() {
  }

  /**
   * Create with an initial list of providers.
   * @param providers provider list.
   */
  public AWSCredentialProviderList(
      Collection<AWSCredentialsProvider> providers) {
    this.providers.addAll(providers);
  }

  /**
   * Create with an initial list of providers.
   * @param name name for error messages, may be ""
   * @param providerArgs provider list.
   */
  public AWSCredentialProviderList(final String name,
      final AWSCredentialsProvider... providerArgs) {
    setName(name);
    Collections.addAll(providers, providerArgs);
  }

  /**
   * Set the name; adds a ": " if needed.
   * @param name name to add, or "" for no name.
   */
  public void setName(final String name) {
    if (!name.isEmpty() && !name.endsWith(": ")) {
      this.name = name + ": ";
    } else {
      this.name = name;
    }
  }

  /**
   * Add a new provider.
   * @param p provider
   */
  public void add(AWSCredentialsProvider p) {
    providers.add(p);
  }

  /**
   * Add all providers from another list to this one.
   * @param other the other list.
   */
  public void addAll(AWSCredentialProviderList other) {
    providers.addAll(other.providers);
  }

  /**
   * Refresh all child entries.
   */
  @Override
  public void refresh() {
    if (isClosed()) {
      return;
    }
    for (AWSCredentialsProvider provider : providers) {
      provider.refresh();
    }
  }

  /**
   * Iterate through the list of providers, to find one with credentials.
   * If {@link #reuseLastProvider} is true, then it is re-used.
   * @return a set of credentials (possibly anonymous), for authenticating.
   */
  @Override
  public AWSCredentials getCredentials() {
    if (isClosed()) {
      LOG.warn(CREDENTIALS_REQUESTED_WHEN_CLOSED);
      throw new NoAuthWithAWSException(name +
          CREDENTIALS_REQUESTED_WHEN_CLOSED);
    }
    checkNotEmpty();
    if (reuseLastProvider && lastProvider != null) {
      return lastProvider.getCredentials();
    }

    AmazonClientException lastException = null;
    for (AWSCredentialsProvider provider : providers) {
      try {
        AWSCredentials credentials = provider.getCredentials();
        Preconditions.checkNotNull(credentials,
            "Null credentials returned by %s", provider);
        if ((credentials.getAWSAccessKeyId() != null &&
            credentials.getAWSSecretKey() != null)
            || (credentials instanceof AnonymousAWSCredentials)) {
          lastProvider = provider;
          LOG.debug("Using credentials from {}", provider);
          return credentials;
        }
      } catch (NoAwsCredentialsException e) {
        // don't bother with the stack trace here as it is usually a
        // minor detail.

        // only update the last exception if it isn't set.
        // Why so? Stops delegation token issues being lost on the fallback
        // values.
        if (lastException == null) {
          lastException = e;
        }
        LOG.debug("No credentials from {}: {}",
            provider, e.toString());
      } catch (AmazonClientException e) {
        lastException = e;
        LOG.debug("No credentials provided by {}: {}",
            provider, e.toString(), e);
      }
    }

    // no providers had any credentials. Rethrow the last exception
    // or create a new one.
    String message =  name +  "No AWS Credentials provided by "
        + listProviderNames();
    if (lastException != null) {
      message += ": " + lastException;
    }
    if (lastException instanceof CredentialInitializationException) {
      throw lastException;
    } else {
      throw new NoAuthWithAWSException(message, lastException);
    }
  }

  /**
   * Returns the underlying list of providers.
   *
   * @return providers
   */
  @VisibleForTesting
  List<AWSCredentialsProvider> getProviders() {
    return providers;
  }

  /**
   * Verify that the provider list is not empty.
   * @throws AmazonClientException if there are no providers.
   */
  public void checkNotEmpty() {
    if (providers.isEmpty()) {
      throw new NoAuthWithAWSException(name + NO_AWS_CREDENTIAL_PROVIDERS);
    }
  }

  /**
   * List all the providers' names.
   * @return a list of names, separated by spaces (with a trailing one).
   * If there are no providers, "" is returned.
   */
  public String listProviderNames() {
    return providers.stream()
        .map(provider -> provider.getClass().getSimpleName() + ' ')
        .collect(Collectors.joining());
  }

  /**
   * The string value is this class name and the string values of nested
   * providers.
   * @return a string value for debugging.
   */
  @Override
  public String toString() {
    return "AWSCredentialProviderList[" +
        name +
        "refcount= " + refCount.get() + ": [" +
        StringUtils.join(providers, ", ") + ']'
        + (lastProvider != null ? (" last provider: " + lastProvider) : "");
  }

  /**
   * Get a reference to this object with an updated reference count.
   *
   * @return a reference to this
   */
  public synchronized AWSCredentialProviderList share() {
    Preconditions.checkState(!closed.get(), "Provider list is closed");
    refCount.incrementAndGet();
    return this;
  }

  /**
   * Get the current reference count.
   * @return the current ref count
   */
  @VisibleForTesting
  public int getRefCount() {
    return refCount.get();
  }

  /**
   * Get the closed flag.
   * @return true iff the list is closed.
   */
  @VisibleForTesting
  public boolean isClosed() {
    return closed.get();
  }

  /**
   * Close routine will close all providers in the list which implement
   * {@code Closeable}.
   * This matters because some providers start a background thread to
   * refresh their secrets.
   */
  @Override
  public void close() {
    synchronized (this) {
      if (closed.get()) {
        // already closed: no-op
        return;
      }
      int remainder = refCount.decrementAndGet();
      if (remainder != 0) {
        // still actively used, or somehow things are
        // now negative
        LOG.debug("Not closing {}", this);
        return;
      }
      // at this point, the closing is going to happen
      LOG.debug("Closing {}", this);
      closed.set(true);
    }

    // do this outside the synchronized block.
    for (AWSCredentialsProvider p : providers) {
      if (p instanceof Closeable) {
        IOUtils.closeStream((Closeable) p);
      } else if (p instanceof AutoCloseable) {
        S3AUtils.closeAutocloseables(LOG, (AutoCloseable)p);
      }
    }
  }

  /**
   * Get the size of this list.
   * @return the number of providers in the list.
   */
  public int size() {
    return providers.size();
  }
}

相关信息

hadoop 源码目录

相关文章

hadoop AWSBadRequestException 源码

hadoop AWSClientIOException 源码

hadoop AWSNoResponseException 源码

hadoop AWSRedirectException 源码

hadoop AWSS3IOException 源码

hadoop AWSServiceIOException 源码

hadoop AWSServiceThrottledException 源码

hadoop AWSStatus500Exception 源码

hadoop AnonymousAWSCredentialsProvider 源码

hadoop ArnResource 源码