array( 'Amazon S3', 's3.{region}.amazon' . 'aws.com', 'us-east-1', false ),
'b2' => array( 'Backblaze B2', 's3.{region}.backblazeb2.com', 'us-west-004', true ),
'r2' => array( 'Cloudflare R2', '{account}.r2.cloudflarestorage.com', 'auto', true ),
'wasabi' => array( 'Wasabi', 's3.{region}.wasabisys.com', 'us-east-1', true ),
'custom' => array( 'Other (S3-compatible)', '', 'us-east-1', true ),
);
/**
* Settings: provider, region, endpoint (custom/R2), bucket, access_key, secret_key, folder.
*
* @var array
*/
private $s;
/**
* Fixed time for tests (Unix), or null.
*
* @var int|null
*/
public $now = null;
/**
* Creates the client.
*
* @param array|null $settings Settings (default: saved ones).
*/
public function __construct( $settings = null ) {
$this->s = array_merge(
array(
'provider' => 'aws',
'region' => '',
'endpoint' => '',
'bucket' => '',
'access_key' => '',
'secret_key' => '',
'folder' => 'Site Replica',
),
is_array( $settings ) ? $settings : Destinations::get( 's3' )
);
}
/**
* Short ID.
*/
public function id() {
return 's3';
}
/**
* Name shown to people.
*/
public function label() {
$p = self::PROVIDERS[ $this->s['provider'] ] ?? self::PROVIDERS['custom'];
return '' !== $this->s['bucket'] ? $p[0] . ' (' . $this->s['bucket'] . ')' : $p[0];
}
/**
* Set up.
*/
public function ready() {
return '' !== $this->s['bucket'] && '' !== $this->s['access_key'] && '' !== $this->s['secret_key'] && '' !== $this->base_url();
}
/**
* Top folder name.
*/
public function root_name() {
return '' !== trim( (string) $this->s['folder'], '/ ' ) ? trim( (string) $this->s['folder'], '/ ' ) : 'Site Replica';
}
/**
* The bucket's top level.
*/
public function top() {
return '';
}
/**
* Region used for signing.
*/
private function region() {
$p = self::PROVIDERS[ $this->s['provider'] ] ?? self::PROVIDERS['custom'];
return '' !== trim( (string) $this->s['region'] ) ? trim( (string) $this->s['region'] ) : $p[2];
}
/**
* Base address of the bucket (scheme, host and, for path-style, the bucket).
*/
public function base_url() {
$p = self::PROVIDERS[ $this->s['provider'] ] ?? self::PROVIDERS['custom'];
$bucket = (string) $this->s['bucket'];
if ( 'custom' === $this->s['provider'] || 'r2' === $this->s['provider'] ) {
$host = trim( (string) $this->s['endpoint'] );
if ( '' === $host ) {
return '';
}
if ( ! preg_match( '#^https?://#', $host ) ) {
$host = 'https://' . $host;
}
return rtrim( $host, '/' ) . '/' . rawurlencode( $bucket );
}
$host = str_replace( '{region}', $this->region(), $p[1] );
return $p[3] ? 'https://' . $host . '/' . rawurlencode( $bucket ) : 'https://' . $bucket . '.' . $host;
}
/* -------------------------------------------------------------------------
* Destination
* ---------------------------------------------------------------------- */
/**
* Bucket facts (also the connection test).
*
* @return array
*/
public function about() {
$this->request( 'GET', '', array( 'list-type' => '2', 'max-keys' => '1' ) );
return array(
'email' => '',
'name' => (string) $this->s['bucket'],
'limit' => null,
'usage' => 0,
);
}
/**
* Folders are prefixes: nothing to create.
*
* @param string[] $names Folder names.
* @return string Prefix ending in /.
*/
public function folder( array $names ) {
return implode( '/', array_map( array( __CLASS__, 'clean_name' ), $names ) ) . '/';
}
/**
* Finds a child.
*
* @param string $parent Prefix ('' for the top).
* @param string $name Name.
* @param bool $folder Only folders.
* @return array|null
*/
public function find( $parent, $name, $folder = false ) {
$key = $parent . self::clean_name( $name );
if ( $folder ) {
$list = $this->list_objects( $key . '/', '', 1 );
return $list['files'] || $list['folders'] ? self::folder_entry( $key . '/' ) : null;
}
$response = $this->request( 'HEAD', $key, array(), '', array(), array( 404 ) );
if ( 404 === (int) wp_remote_retrieve_response_code( $response ) ) {
return null;
}
$etag = trim( (string) wp_remote_retrieve_header( $response, 'etag' ), '"' );
$md5 = (string) wp_remote_retrieve_header( $response, 'x-amz-meta-md5' );
return array(
'id' => $key,
'name' => (string) $name,
'size' => (string) (int) wp_remote_retrieve_header( $response, 'content-length' ),
'md5Checksum' => '' !== $md5 ? $md5 : ( false === strpos( $etag, '-' ) ? $etag : '' ),
'createdTime' => gmdate( 'c', (int) strtotime( (string) wp_remote_retrieve_header( $response, 'last-modified' ) ) ),
'mimeType' => (string) wp_remote_retrieve_header( $response, 'content-type' ),
);
}
/**
* Children of a prefix.
*
* @param string $parent Prefix.
* @param bool $folders_only Only folders.
* @return array[]
*/
public function children( $parent, $folders_only = false ) {
$list = $this->list_objects( $parent, '/', 0 );
$out = array();
foreach ( $list['folders'] as $prefix ) {
$out[] = self::folder_entry( $prefix );
}
if ( ! $folders_only ) {
foreach ( $list['files'] as $f ) {
if ( $f['key'] === $parent ) {
continue;
}
$out[] = array(
'id' => $f['key'],
'name' => substr( $f['key'], strlen( $parent ) ),
'size' => (string) $f['size'],
'md5Checksum' => false === strpos( $f['etag'], '-' ) ? $f['etag'] : '',
'createdTime' => $f['modified'],
'mimeType' => 'application/octet-stream',
);
}
}
usort(
$out,
static function ( $a, $b ) {
return strcmp( $a['name'], $b['name'] );
}
);
return $out;
}
/**
* Uploads a small object.
*
* @param string $parent Prefix.
* @param string $name Name.
* @param string $data Contents.
* @param string $mime MIME type.
* @return array
*/
public function upload_small( $parent, $name, $data, $mime = 'application/octet-stream' ) {
$key = $parent . self::clean_name( $name );
$md5 = md5( $data );
$this->request(
'PUT',
$key,
array(),
$data,
array(
'Content-Type' => $mime,
'Content-MD5' => base64_encode( md5( $data, true ) ), // phpcs:ignore WordPress.PHP.DiscouragedPHPFunctions.obfuscation_base64_encode -- Required header format.
'x-amz-meta-md5' => $md5,
)
);
return array(
'id' => $key,
'name' => (string) $name,
'size' => (string) strlen( $data ),
'md5Checksum' => $md5,
);
}
/**
* Starts a multipart upload.
*
* @param string $parent Prefix.
* @param string $name Name.
* @param int $size Total size.
* @param string $mime MIME type.
* @return array Session {key, upload, parts: [[n, etag, size]]}.
* @throws Remote_Exception When the storage doesn't start it.
*/
public function upload_start( $parent, $name, $size, $mime = 'application/zip' ) {
$key = $parent . self::clean_name( $name );
$response = $this->request( 'POST', $key, array( 'uploads' => '' ), '', array( 'Content-Type' => $mime ) );
$upload = self::xml_value( (string) wp_remote_retrieve_body( $response ), 'UploadId' );
if ( '' === $upload ) {
throw new Remote_Exception( 's3_upload', __( 'The storage did not start the upload.', 'accessnow-site-replica' ), true, 20 );
}
return array(
'key' => $key,
'upload' => $upload,
'parts' => array(),
);
}
/**
* Uploads one piece; completes the upload after the last one.
*
* @param array $session Session.
* @param string $data Bytes.
* @param int $offset Where the piece starts.
* @param int $total Total size.
* @return array{next: int, file: array|null, session: array}
* @throws Remote_Exception On failure.
*/
public function upload_chunk( $session, $data, $offset, $total ) {
$done = self::parts_size( $session );
if ( (int) $offset !== $done ) {
return $this->upload_status( $session, $total ); // Out of step: say where to continue.
}
$n = count( $session['parts'] ) + 1;
$response = $this->request(
'PUT',
$session['key'],
array(
'partNumber' => (string) $n,
'uploadId' => $session['upload'],
),
$data,
array( 'Content-MD5' => base64_encode( md5( $data, true ) ) ), // phpcs:ignore WordPress.PHP.DiscouragedPHPFunctions.obfuscation_base64_encode -- Required header format.
array( 404 )
);
if ( 404 === (int) wp_remote_retrieve_response_code( $response ) ) {
throw new Remote_Exception( 'drive_session_gone', __( 'The storage forgot an unfinished upload. That part will be uploaded again.', 'accessnow-site-replica' ), true, 3 );
}
$session['parts'][] = array( $n, trim( (string) wp_remote_retrieve_header( $response, 'etag' ), '"' ), strlen( $data ) );
$done += strlen( $data );
if ( $done < (int) $total ) {
return array(
'next' => $done,
'file' => null,
'session' => $session,
);
}
return array(
'next' => $done,
'file' => $this->complete( $session, $total ),
'session' => $session,
);
}
/**
* Asks the storage which pieces arrived.
*
* @param array $session Session.
* @param int $total Total size.
* @return array{next: int, file: array|null, session: array}
* @throws Remote_Exception When the upload is gone.
*/
public function upload_status( $session, $total ) {
$response = $this->request( 'GET', $session['key'], array( 'uploadId' => $session['upload'] ), '', array(), array( 404 ) );
if ( 404 === (int) wp_remote_retrieve_response_code( $response ) ) {
// Finished already (the completion's answer got lost)?
$file = $this->find( dirname( $session['key'] ) . '/', basename( $session['key'] ) );
if ( $file && (int) $file['size'] === (int) $total ) {
return array(
'next' => (int) $total,
'file' => $file,
'session' => $session,
);
}
throw new Remote_Exception( 'drive_session_gone', __( 'The storage forgot an unfinished upload. That part will be uploaded again.', 'accessnow-site-replica' ), true, 3 );
}
$body = (string) wp_remote_retrieve_body( $response );
preg_match_all( '#(.*?)#s', $body, $m );
$parts = array();
foreach ( $m[1] as $part ) {
$parts[] = array( (int) self::xml_value( $part, 'PartNumber' ), trim( self::xml_value( $part, 'ETag' ), '"' ), (int) self::xml_value( $part, 'Size' ) );
}
usort(
$parts,
static function ( $a, $b ) {
return $a[0] <=> $b[0];
}
);
// Only an unbroken run of pieces from the start counts.
$kept = array();
foreach ( $parts as $i => $p ) {
if ( $p[0] !== $i + 1 ) {
break;
}
$kept[] = $p;
}
$session['parts'] = $kept;
$done = self::parts_size( $session );
return array(
'next' => $done,
'file' => $done >= (int) $total && $kept ? $this->complete( $session, $total ) : null,
'session' => $session,
);
}
/**
* Downloads an object or a range of it.
*
* @param string $id Key.
* @param int $offset Start.
* @param int|null $length Length.
* @return string
*/
public function download( $id, $offset = 0, $length = null ) {
$headers = array();
if ( $offset > 0 || null !== $length ) {
$headers['Range'] = 'bytes=' . (int) $offset . '-' . ( null === $length ? '' : (int) $offset + (int) $length - 1 );
}
return (string) wp_remote_retrieve_body( $this->request( 'GET', $id, array(), '', $headers, array(), 120 ) );
}
/**
* Deletes an object, or everything under a prefix (including unfinished uploads).
*
* @param string $id Key or prefix.
*/
public function delete( $id ) {
if ( '/' !== substr( $id, -1 ) ) {
$this->request( 'DELETE', $id, array(), '', array(), array( 404 ) );
return;
}
$token = '';
do {
$list = $this->list_objects( $id, '', 1000, $token );
foreach ( $list['files'] as $f ) {
$this->request( 'DELETE', $f['key'], array(), '', array(), array( 404 ) );
}
$token = $list['next'];
} while ( '' !== $token );
// Unfinished uploads under the prefix. Some storage (MinIO) only lists them by exact key, so
// list them all and pick ours (there are only ever a few).
$response = $this->request( 'GET', '', array( 'uploads' => '' ) );
preg_match_all( '#(.*?)#s', (string) wp_remote_retrieve_body( $response ), $m );
foreach ( $m[1] as $upload ) {
$key = self::xml_value( $upload, 'Key' );
if ( 0 === strpos( $key, $id ) ) {
$this->request( 'DELETE', $key, array( 'uploadId' => self::xml_value( $upload, 'UploadId' ) ), '', array(), array( 404 ) );
}
}
}
/* -------------------------------------------------------------------------
* Helpers
* ---------------------------------------------------------------------- */
/**
* Completes a multipart upload.
*
* @param array $session Session.
* @param int $total Total size.
* @return array File.
*/
private function complete( array $session, $total ) {
$xml = '';
foreach ( $session['parts'] as $p ) {
$xml .= '' . (int) $p[0] . '"' . esc_html( $p[1] ) . '"';
}
$xml .= '';
$response = $this->request( 'POST', $session['key'], array( 'uploadId' => $session['upload'] ), $xml, array( 'Content-Type' => 'application/xml' ) );
$body = (string) wp_remote_retrieve_body( $response );
if ( false !== strpos( $body, '' ) ) {
// Some storage answers 200 with an error body when completing.
throw new Remote_Exception( 's3_complete', sprintf( /* translators: %s: storage message. */ __( 'The storage could not finish the upload: %s', 'accessnow-site-replica' ), self::xml_value( $body, 'Message' ) ), true, 10 );
}
return array(
'id' => $session['key'],
'name' => basename( $session['key'] ),
'size' => (string) (int) $total,
);
}
/**
* Bytes in the pieces uploaded so far.
*
* @param array $session Session.
*/
private static function parts_size( array $session ) {
$n = 0;
foreach ( (array) $session['parts'] as $p ) {
$n += (int) $p[2];
}
return $n;
}
/**
* ListObjectsV2.
*
* @param string $prefix Prefix.
* @param string $delimiter Delimiter ('' for everything).
* @param int $max Most keys (0: all, following pages).
* @param string $token Continuation token.
* @return array{files: array[], folders: string[], next: string}
*/
private function list_objects( $prefix, $delimiter, $max = 0, $token = '' ) {
$files = array();
$folders = array();
do {
$query = array(
'list-type' => '2',
'prefix' => $prefix,
'encoding-type' => 'url',
'max-keys' => (string) ( $max > 0 ? $max : 1000 ),
);
if ( '' !== $delimiter ) {
$query['delimiter'] = $delimiter;
}
if ( '' !== $token ) {
$query['continuation-token'] = $token;
}
$body = (string) wp_remote_retrieve_body( $this->request( 'GET', '', $query ) );
preg_match_all( '#(.*?)#s', $body, $m );
foreach ( $m[1] as $c ) {
$files[] = array(
'key' => urldecode( self::xml_value( $c, 'Key' ) ),
'size' => (int) self::xml_value( $c, 'Size' ),
'etag' => trim( self::xml_value( $c, 'ETag' ), '"' ),
'modified' => gmdate( 'c', (int) strtotime( self::xml_value( $c, 'LastModified' ) ) ),
);
}
preg_match_all( '#(.*?)#s', $body, $m );
foreach ( $m[1] as $c ) {
$folders[] = urldecode( self::xml_value( $c, 'Prefix' ) );
}
$token = 'true' === self::xml_value( $body, 'IsTruncated' ) ? self::xml_value( $body, 'NextContinuationToken' ) : '';
} while ( 0 === $max && '' !== $token );
return array(
'files' => $files,
'folders' => $folders,
'next' => $token,
);
}
/**
* Entry for a prefix. Backup folders are named after their date, which serves as creation time.
*
* @param string $prefix Prefix ending in /.
*/
private static function folder_entry( $prefix ) {
$name = basename( rtrim( $prefix, '/' ) );
$created = preg_match( '/^(\d{4})(\d{2})(\d{2})-(\d{2})(\d{2})(\d{2})-/', $name, $m ) ? sprintf( '%s-%s-%sT%s:%s:%sZ', $m[1], $m[2], $m[3], $m[4], $m[5], $m[6] ) : '';
return array(
'id' => $prefix,
'name' => $name,
'mimeType' => 'application/vnd.google-apps.folder',
'createdTime' => $created,
);
}
/**
* A folder or file name made safe as part of a key.
*
* @param string $name Name.
*/
private static function clean_name( $name ) {
return trim( str_replace( array( '/', '\\' ), '-', (string) $name ) );
}
/**
* Text of the first element with this name.
*
* @param string $xml XML.
* @param string $name Element.
*/
private static function xml_value( $xml, $name ) {
return preg_match( '#<' . $name . '>(.*?)' . $name . '>#s', $xml, $m ) ? html_entity_decode( $m[1], ENT_QUOTES | ENT_XML1, 'UTF-8' ) : '';
}
/**
* Signed request.
*
* @param string $method Method.
* @param string $key Object key ('' for the bucket).
* @param array $query Query parameters.
* @param string $body Body.
* @param array $headers Extra headers.
* @param int[] $ok Extra status codes that are not errors.
* @param int $timeout Timeout.
* @return array WordPress HTTP response.
* @throws Remote_Exception On failure.
*/
private function request( $method, $key, array $query = array(), $body = '', array $headers = array(), array $ok = array(), $timeout = 240 ) {
$base = $this->base_url();
if ( '' === $base || ! $this->ready() ) {
throw new Remote_Exception( 's3_not_set', __( 'The S3 storage is not set up. Enter it under Site Replica → Cloud Backups.', 'accessnow-site-replica' ) );
}
$url = $base . ( '' !== $key ? '/' . implode( '/', array_map( 'rawurlencode', explode( '/', $key ) ) ) : '/' );
$signed = $this->sign( $method, $url, $query, $headers, (string) $body );
$qs = self::canonical_query( $query );
if ( function_exists( 'set_time_limit' ) ) {
@set_time_limit( $timeout + 30 ); // phpcs:ignore WordPress.PHP.NoSilencedErrors.Discouraged, Squiz.PHP.DiscouragedFunctions.Discouraged -- Network waits count on some hosts.
}
$args = array(
'method' => $method,
'timeout' => $timeout,
'redirection' => 0,
'headers' => $signed,
'sslverify' => true,
);
if ( '' !== $body || in_array( $method, array( 'PUT', 'POST' ), true ) ) {
$args['body'] = (string) $body;
}
$response = wp_remote_request( $url . ( '' !== $qs ? '?' . $qs : '' ), $args );
if ( is_wp_error( $response ) ) {
throw new Remote_Exception( 's3_network', sprintf( /* translators: %s: error. */ __( 'Could not reach the storage: %s', 'accessnow-site-replica' ), $response->get_error_message() ), true, 20 );
}
$code = (int) wp_remote_retrieve_response_code( $response );
if ( ( $code >= 200 && $code < 300 ) || in_array( $code, $ok, true ) ) {
return $response;
}
$xml = (string) wp_remote_retrieve_body( $response );
$error = self::xml_value( $xml, 'Code' );
$message = self::xml_value( $xml, 'Message' );
if ( 429 === $code || $code >= 500 || in_array( $error, array( 'SlowDown', 'InternalError', 'ServiceUnavailable', 'RequestTimeout' ), true ) ) {
throw new Remote_Exception( 's3_busy', __( 'The storage is busy. Trying again shortly.', 'accessnow-site-replica' ), true, 30 );
}
if ( in_array( $error, array( 'InvalidAccessKeyId', 'SignatureDoesNotMatch', 'AccessDenied', 'AuthorizationHeaderMalformed' ), true ) ) {
/* translators: %s: storage error code. */
throw new Remote_Exception( 's3_auth', sprintf( __( 'The storage refused the access key (%s). Check the key, secret, region and bucket.', 'accessnow-site-replica' ), $error ) );
}
if ( 'NoSuchBucket' === $error ) {
throw new Remote_Exception( 's3_bucket', __( 'That bucket does not exist (check its name and region).', 'accessnow-site-replica' ) );
}
if ( 'BadDigest' === $error || 'InvalidDigest' === $error ) {
throw new Remote_Exception( 'upload_md5', __( 'A piece arrived damaged at the storage. It will be sent again.', 'accessnow-site-replica' ), true, 5 );
}
/* translators: 1: HTTP status, 2: storage error code, 3: storage message. */
throw new Remote_Exception( 's3_http_' . $code, sprintf( __( 'The storage answered %1$d %2$s: %3$s', 'accessnow-site-replica' ), $code, $error, $message ) );
}
/**
* Signature Version 4 headers for a request.
*
* @param string $method Method.
* @param string $url Address without the query.
* @param array $query Query parameters.
* @param array $headers Extra headers.
* @param string $body Body.
* @return array Headers to send (including Authorization).
*/
public function sign( $method, $url, array $query, array $headers, $body ) {
$now = null === $this->now ? time() : (int) $this->now;
$amzdate = gmdate( 'Ymd\THis\Z', $now );
$date = gmdate( 'Ymd', $now );
$parts = wp_parse_url( $url );
$host = $parts['host'] . ( isset( $parts['port'] ) ? ':' . $parts['port'] : '' );
$path = isset( $parts['path'] ) && '' !== $parts['path'] ? $parts['path'] : '/';
$hash = hash( 'sha256', $body );
$all = array(
'host' => $host,
'x-amz-content-sha256' => $hash,
'x-amz-date' => $amzdate,
);
foreach ( $headers as $k => $v ) {
$all[ strtolower( $k ) ] = trim( preg_replace( '/\s+/', ' ', (string) $v ) );
}
ksort( $all );
$canonical_headers = '';
foreach ( $all as $k => $v ) {
$canonical_headers .= $k . ':' . $v . "\n";
}
$signed_headers = implode( ';', array_keys( $all ) );
$canonical = $method . "\n" . $path . "\n" . self::canonical_query( $query ) . "\n" . $canonical_headers . "\n" . $signed_headers . "\n" . $hash;
$scope = $date . '/' . $this->region() . '/s3/aws4_request';
$to_sign = "AWS4-HMAC-SHA256\n" . $amzdate . "\n" . $scope . "\n" . hash( 'sha256', $canonical );
$k = hash_hmac( 'sha256', $date, 'AWS4' . $this->s['secret_key'], true );
$k = hash_hmac( 'sha256', $this->region(), $k, true );
$k = hash_hmac( 'sha256', 's3', $k, true );
$k = hash_hmac( 'sha256', 'aws4_request', $k, true );
$signature = hash_hmac( 'sha256', $to_sign, $k );
$out = array();
foreach ( $all as $k2 => $v ) {
if ( 'host' !== $k2 ) {
$out[ $k2 ] = $v;
}
}
$out['Authorization'] = 'AWS4-HMAC-SHA256 Credential=' . $this->s['access_key'] . '/' . $scope . ', SignedHeaders=' . $signed_headers . ', Signature=' . $signature;
return $out;
}
/**
* Canonical query string (sorted, RFC 3986 encoded, "key=" for empty values).
*
* @param array $query Parameters.
*/
private static function canonical_query( array $query ) {
ksort( $query, SORT_STRING );
$pairs = array();
foreach ( $query as $k => $v ) {
$pairs[] = rawurlencode( (string) $k ) . '=' . rawurlencode( (string) $v );
}
return implode( '&', $pairs );
}
}