array( 'Amazon S3', 's3.{region}.amazon' . 'aws.com', 'us-east-1', false ), 'b2' => array( 'Backblaze B2', 's3.{region}.backblazeb2.com', 'us-west-004', true ), 'r2' => array( 'Cloudflare R2', '{account}.r2.cloudflarestorage.com', 'auto', true ), 'wasabi' => array( 'Wasabi', 's3.{region}.wasabisys.com', 'us-east-1', true ), 'custom' => array( 'Other (S3-compatible)', '', 'us-east-1', true ), ); /** * Settings: provider, region, endpoint (custom/R2), bucket, access_key, secret_key, folder. * * @var array */ private $s; /** * Fixed time for tests (Unix), or null. * * @var int|null */ public $now = null; /** * Creates the client. * * @param array|null $settings Settings (default: saved ones). */ public function __construct( $settings = null ) { $this->s = array_merge( array( 'provider' => 'aws', 'region' => '', 'endpoint' => '', 'bucket' => '', 'access_key' => '', 'secret_key' => '', 'folder' => 'Site Replica', ), is_array( $settings ) ? $settings : Destinations::get( 's3' ) ); } /** * Short ID. */ public function id() { return 's3'; } /** * Name shown to people. */ public function label() { $p = self::PROVIDERS[ $this->s['provider'] ] ?? self::PROVIDERS['custom']; return '' !== $this->s['bucket'] ? $p[0] . ' (' . $this->s['bucket'] . ')' : $p[0]; } /** * Set up. */ public function ready() { return '' !== $this->s['bucket'] && '' !== $this->s['access_key'] && '' !== $this->s['secret_key'] && '' !== $this->base_url(); } /** * Top folder name. */ public function root_name() { return '' !== trim( (string) $this->s['folder'], '/ ' ) ? trim( (string) $this->s['folder'], '/ ' ) : 'Site Replica'; } /** * The bucket's top level. */ public function top() { return ''; } /** * Region used for signing. */ private function region() { $p = self::PROVIDERS[ $this->s['provider'] ] ?? self::PROVIDERS['custom']; return '' !== trim( (string) $this->s['region'] ) ? trim( (string) $this->s['region'] ) : $p[2]; } /** * Base address of the bucket (scheme, host and, for path-style, the bucket). */ public function base_url() { $p = self::PROVIDERS[ $this->s['provider'] ] ?? self::PROVIDERS['custom']; $bucket = (string) $this->s['bucket']; if ( 'custom' === $this->s['provider'] || 'r2' === $this->s['provider'] ) { $host = trim( (string) $this->s['endpoint'] ); if ( '' === $host ) { return ''; } if ( ! preg_match( '#^https?://#', $host ) ) { $host = 'https://' . $host; } return rtrim( $host, '/' ) . '/' . rawurlencode( $bucket ); } $host = str_replace( '{region}', $this->region(), $p[1] ); return $p[3] ? 'https://' . $host . '/' . rawurlencode( $bucket ) : 'https://' . $bucket . '.' . $host; } /* ------------------------------------------------------------------------- * Destination * ---------------------------------------------------------------------- */ /** * Bucket facts (also the connection test). * * @return array */ public function about() { $this->request( 'GET', '', array( 'list-type' => '2', 'max-keys' => '1' ) ); return array( 'email' => '', 'name' => (string) $this->s['bucket'], 'limit' => null, 'usage' => 0, ); } /** * Folders are prefixes: nothing to create. * * @param string[] $names Folder names. * @return string Prefix ending in /. */ public function folder( array $names ) { return implode( '/', array_map( array( __CLASS__, 'clean_name' ), $names ) ) . '/'; } /** * Finds a child. * * @param string $parent Prefix ('' for the top). * @param string $name Name. * @param bool $folder Only folders. * @return array|null */ public function find( $parent, $name, $folder = false ) { $key = $parent . self::clean_name( $name ); if ( $folder ) { $list = $this->list_objects( $key . '/', '', 1 ); return $list['files'] || $list['folders'] ? self::folder_entry( $key . '/' ) : null; } $response = $this->request( 'HEAD', $key, array(), '', array(), array( 404 ) ); if ( 404 === (int) wp_remote_retrieve_response_code( $response ) ) { return null; } $etag = trim( (string) wp_remote_retrieve_header( $response, 'etag' ), '"' ); $md5 = (string) wp_remote_retrieve_header( $response, 'x-amz-meta-md5' ); return array( 'id' => $key, 'name' => (string) $name, 'size' => (string) (int) wp_remote_retrieve_header( $response, 'content-length' ), 'md5Checksum' => '' !== $md5 ? $md5 : ( false === strpos( $etag, '-' ) ? $etag : '' ), 'createdTime' => gmdate( 'c', (int) strtotime( (string) wp_remote_retrieve_header( $response, 'last-modified' ) ) ), 'mimeType' => (string) wp_remote_retrieve_header( $response, 'content-type' ), ); } /** * Children of a prefix. * * @param string $parent Prefix. * @param bool $folders_only Only folders. * @return array[] */ public function children( $parent, $folders_only = false ) { $list = $this->list_objects( $parent, '/', 0 ); $out = array(); foreach ( $list['folders'] as $prefix ) { $out[] = self::folder_entry( $prefix ); } if ( ! $folders_only ) { foreach ( $list['files'] as $f ) { if ( $f['key'] === $parent ) { continue; } $out[] = array( 'id' => $f['key'], 'name' => substr( $f['key'], strlen( $parent ) ), 'size' => (string) $f['size'], 'md5Checksum' => false === strpos( $f['etag'], '-' ) ? $f['etag'] : '', 'createdTime' => $f['modified'], 'mimeType' => 'application/octet-stream', ); } } usort( $out, static function ( $a, $b ) { return strcmp( $a['name'], $b['name'] ); } ); return $out; } /** * Uploads a small object. * * @param string $parent Prefix. * @param string $name Name. * @param string $data Contents. * @param string $mime MIME type. * @return array */ public function upload_small( $parent, $name, $data, $mime = 'application/octet-stream' ) { $key = $parent . self::clean_name( $name ); $md5 = md5( $data ); $this->request( 'PUT', $key, array(), $data, array( 'Content-Type' => $mime, 'Content-MD5' => base64_encode( md5( $data, true ) ), // phpcs:ignore WordPress.PHP.DiscouragedPHPFunctions.obfuscation_base64_encode -- Required header format. 'x-amz-meta-md5' => $md5, ) ); return array( 'id' => $key, 'name' => (string) $name, 'size' => (string) strlen( $data ), 'md5Checksum' => $md5, ); } /** * Starts a multipart upload. * * @param string $parent Prefix. * @param string $name Name. * @param int $size Total size. * @param string $mime MIME type. * @return array Session {key, upload, parts: [[n, etag, size]]}. * @throws Remote_Exception When the storage doesn't start it. */ public function upload_start( $parent, $name, $size, $mime = 'application/zip' ) { $key = $parent . self::clean_name( $name ); $response = $this->request( 'POST', $key, array( 'uploads' => '' ), '', array( 'Content-Type' => $mime ) ); $upload = self::xml_value( (string) wp_remote_retrieve_body( $response ), 'UploadId' ); if ( '' === $upload ) { throw new Remote_Exception( 's3_upload', __( 'The storage did not start the upload.', 'accessnow-site-replica' ), true, 20 ); } return array( 'key' => $key, 'upload' => $upload, 'parts' => array(), ); } /** * Uploads one piece; completes the upload after the last one. * * @param array $session Session. * @param string $data Bytes. * @param int $offset Where the piece starts. * @param int $total Total size. * @return array{next: int, file: array|null, session: array} * @throws Remote_Exception On failure. */ public function upload_chunk( $session, $data, $offset, $total ) { $done = self::parts_size( $session ); if ( (int) $offset !== $done ) { return $this->upload_status( $session, $total ); // Out of step: say where to continue. } $n = count( $session['parts'] ) + 1; $response = $this->request( 'PUT', $session['key'], array( 'partNumber' => (string) $n, 'uploadId' => $session['upload'], ), $data, array( 'Content-MD5' => base64_encode( md5( $data, true ) ) ), // phpcs:ignore WordPress.PHP.DiscouragedPHPFunctions.obfuscation_base64_encode -- Required header format. array( 404 ) ); if ( 404 === (int) wp_remote_retrieve_response_code( $response ) ) { throw new Remote_Exception( 'drive_session_gone', __( 'The storage forgot an unfinished upload. That part will be uploaded again.', 'accessnow-site-replica' ), true, 3 ); } $session['parts'][] = array( $n, trim( (string) wp_remote_retrieve_header( $response, 'etag' ), '"' ), strlen( $data ) ); $done += strlen( $data ); if ( $done < (int) $total ) { return array( 'next' => $done, 'file' => null, 'session' => $session, ); } return array( 'next' => $done, 'file' => $this->complete( $session, $total ), 'session' => $session, ); } /** * Asks the storage which pieces arrived. * * @param array $session Session. * @param int $total Total size. * @return array{next: int, file: array|null, session: array} * @throws Remote_Exception When the upload is gone. */ public function upload_status( $session, $total ) { $response = $this->request( 'GET', $session['key'], array( 'uploadId' => $session['upload'] ), '', array(), array( 404 ) ); if ( 404 === (int) wp_remote_retrieve_response_code( $response ) ) { // Finished already (the completion's answer got lost)? $file = $this->find( dirname( $session['key'] ) . '/', basename( $session['key'] ) ); if ( $file && (int) $file['size'] === (int) $total ) { return array( 'next' => (int) $total, 'file' => $file, 'session' => $session, ); } throw new Remote_Exception( 'drive_session_gone', __( 'The storage forgot an unfinished upload. That part will be uploaded again.', 'accessnow-site-replica' ), true, 3 ); } $body = (string) wp_remote_retrieve_body( $response ); preg_match_all( '#(.*?)#s', $body, $m ); $parts = array(); foreach ( $m[1] as $part ) { $parts[] = array( (int) self::xml_value( $part, 'PartNumber' ), trim( self::xml_value( $part, 'ETag' ), '"' ), (int) self::xml_value( $part, 'Size' ) ); } usort( $parts, static function ( $a, $b ) { return $a[0] <=> $b[0]; } ); // Only an unbroken run of pieces from the start counts. $kept = array(); foreach ( $parts as $i => $p ) { if ( $p[0] !== $i + 1 ) { break; } $kept[] = $p; } $session['parts'] = $kept; $done = self::parts_size( $session ); return array( 'next' => $done, 'file' => $done >= (int) $total && $kept ? $this->complete( $session, $total ) : null, 'session' => $session, ); } /** * Downloads an object or a range of it. * * @param string $id Key. * @param int $offset Start. * @param int|null $length Length. * @return string */ public function download( $id, $offset = 0, $length = null ) { $headers = array(); if ( $offset > 0 || null !== $length ) { $headers['Range'] = 'bytes=' . (int) $offset . '-' . ( null === $length ? '' : (int) $offset + (int) $length - 1 ); } return (string) wp_remote_retrieve_body( $this->request( 'GET', $id, array(), '', $headers, array(), 120 ) ); } /** * Deletes an object, or everything under a prefix (including unfinished uploads). * * @param string $id Key or prefix. */ public function delete( $id ) { if ( '/' !== substr( $id, -1 ) ) { $this->request( 'DELETE', $id, array(), '', array(), array( 404 ) ); return; } $token = ''; do { $list = $this->list_objects( $id, '', 1000, $token ); foreach ( $list['files'] as $f ) { $this->request( 'DELETE', $f['key'], array(), '', array(), array( 404 ) ); } $token = $list['next']; } while ( '' !== $token ); // Unfinished uploads under the prefix. Some storage (MinIO) only lists them by exact key, so // list them all and pick ours (there are only ever a few). $response = $this->request( 'GET', '', array( 'uploads' => '' ) ); preg_match_all( '#(.*?)#s', (string) wp_remote_retrieve_body( $response ), $m ); foreach ( $m[1] as $upload ) { $key = self::xml_value( $upload, 'Key' ); if ( 0 === strpos( $key, $id ) ) { $this->request( 'DELETE', $key, array( 'uploadId' => self::xml_value( $upload, 'UploadId' ) ), '', array(), array( 404 ) ); } } } /* ------------------------------------------------------------------------- * Helpers * ---------------------------------------------------------------------- */ /** * Completes a multipart upload. * * @param array $session Session. * @param int $total Total size. * @return array File. */ private function complete( array $session, $total ) { $xml = ''; foreach ( $session['parts'] as $p ) { $xml .= '' . (int) $p[0] . '"' . esc_html( $p[1] ) . '"'; } $xml .= ''; $response = $this->request( 'POST', $session['key'], array( 'uploadId' => $session['upload'] ), $xml, array( 'Content-Type' => 'application/xml' ) ); $body = (string) wp_remote_retrieve_body( $response ); if ( false !== strpos( $body, '' ) ) { // Some storage answers 200 with an error body when completing. throw new Remote_Exception( 's3_complete', sprintf( /* translators: %s: storage message. */ __( 'The storage could not finish the upload: %s', 'accessnow-site-replica' ), self::xml_value( $body, 'Message' ) ), true, 10 ); } return array( 'id' => $session['key'], 'name' => basename( $session['key'] ), 'size' => (string) (int) $total, ); } /** * Bytes in the pieces uploaded so far. * * @param array $session Session. */ private static function parts_size( array $session ) { $n = 0; foreach ( (array) $session['parts'] as $p ) { $n += (int) $p[2]; } return $n; } /** * ListObjectsV2. * * @param string $prefix Prefix. * @param string $delimiter Delimiter ('' for everything). * @param int $max Most keys (0: all, following pages). * @param string $token Continuation token. * @return array{files: array[], folders: string[], next: string} */ private function list_objects( $prefix, $delimiter, $max = 0, $token = '' ) { $files = array(); $folders = array(); do { $query = array( 'list-type' => '2', 'prefix' => $prefix, 'encoding-type' => 'url', 'max-keys' => (string) ( $max > 0 ? $max : 1000 ), ); if ( '' !== $delimiter ) { $query['delimiter'] = $delimiter; } if ( '' !== $token ) { $query['continuation-token'] = $token; } $body = (string) wp_remote_retrieve_body( $this->request( 'GET', '', $query ) ); preg_match_all( '#(.*?)#s', $body, $m ); foreach ( $m[1] as $c ) { $files[] = array( 'key' => urldecode( self::xml_value( $c, 'Key' ) ), 'size' => (int) self::xml_value( $c, 'Size' ), 'etag' => trim( self::xml_value( $c, 'ETag' ), '"' ), 'modified' => gmdate( 'c', (int) strtotime( self::xml_value( $c, 'LastModified' ) ) ), ); } preg_match_all( '#(.*?)#s', $body, $m ); foreach ( $m[1] as $c ) { $folders[] = urldecode( self::xml_value( $c, 'Prefix' ) ); } $token = 'true' === self::xml_value( $body, 'IsTruncated' ) ? self::xml_value( $body, 'NextContinuationToken' ) : ''; } while ( 0 === $max && '' !== $token ); return array( 'files' => $files, 'folders' => $folders, 'next' => $token, ); } /** * Entry for a prefix. Backup folders are named after their date, which serves as creation time. * * @param string $prefix Prefix ending in /. */ private static function folder_entry( $prefix ) { $name = basename( rtrim( $prefix, '/' ) ); $created = preg_match( '/^(\d{4})(\d{2})(\d{2})-(\d{2})(\d{2})(\d{2})-/', $name, $m ) ? sprintf( '%s-%s-%sT%s:%s:%sZ', $m[1], $m[2], $m[3], $m[4], $m[5], $m[6] ) : ''; return array( 'id' => $prefix, 'name' => $name, 'mimeType' => 'application/vnd.google-apps.folder', 'createdTime' => $created, ); } /** * A folder or file name made safe as part of a key. * * @param string $name Name. */ private static function clean_name( $name ) { return trim( str_replace( array( '/', '\\' ), '-', (string) $name ) ); } /** * Text of the first element with this name. * * @param string $xml XML. * @param string $name Element. */ private static function xml_value( $xml, $name ) { return preg_match( '#<' . $name . '>(.*?)#s', $xml, $m ) ? html_entity_decode( $m[1], ENT_QUOTES | ENT_XML1, 'UTF-8' ) : ''; } /** * Signed request. * * @param string $method Method. * @param string $key Object key ('' for the bucket). * @param array $query Query parameters. * @param string $body Body. * @param array $headers Extra headers. * @param int[] $ok Extra status codes that are not errors. * @param int $timeout Timeout. * @return array WordPress HTTP response. * @throws Remote_Exception On failure. */ private function request( $method, $key, array $query = array(), $body = '', array $headers = array(), array $ok = array(), $timeout = 240 ) { $base = $this->base_url(); if ( '' === $base || ! $this->ready() ) { throw new Remote_Exception( 's3_not_set', __( 'The S3 storage is not set up. Enter it under Site Replica → Cloud Backups.', 'accessnow-site-replica' ) ); } $url = $base . ( '' !== $key ? '/' . implode( '/', array_map( 'rawurlencode', explode( '/', $key ) ) ) : '/' ); $signed = $this->sign( $method, $url, $query, $headers, (string) $body ); $qs = self::canonical_query( $query ); if ( function_exists( 'set_time_limit' ) ) { @set_time_limit( $timeout + 30 ); // phpcs:ignore WordPress.PHP.NoSilencedErrors.Discouraged, Squiz.PHP.DiscouragedFunctions.Discouraged -- Network waits count on some hosts. } $args = array( 'method' => $method, 'timeout' => $timeout, 'redirection' => 0, 'headers' => $signed, 'sslverify' => true, ); if ( '' !== $body || in_array( $method, array( 'PUT', 'POST' ), true ) ) { $args['body'] = (string) $body; } $response = wp_remote_request( $url . ( '' !== $qs ? '?' . $qs : '' ), $args ); if ( is_wp_error( $response ) ) { throw new Remote_Exception( 's3_network', sprintf( /* translators: %s: error. */ __( 'Could not reach the storage: %s', 'accessnow-site-replica' ), $response->get_error_message() ), true, 20 ); } $code = (int) wp_remote_retrieve_response_code( $response ); if ( ( $code >= 200 && $code < 300 ) || in_array( $code, $ok, true ) ) { return $response; } $xml = (string) wp_remote_retrieve_body( $response ); $error = self::xml_value( $xml, 'Code' ); $message = self::xml_value( $xml, 'Message' ); if ( 429 === $code || $code >= 500 || in_array( $error, array( 'SlowDown', 'InternalError', 'ServiceUnavailable', 'RequestTimeout' ), true ) ) { throw new Remote_Exception( 's3_busy', __( 'The storage is busy. Trying again shortly.', 'accessnow-site-replica' ), true, 30 ); } if ( in_array( $error, array( 'InvalidAccessKeyId', 'SignatureDoesNotMatch', 'AccessDenied', 'AuthorizationHeaderMalformed' ), true ) ) { /* translators: %s: storage error code. */ throw new Remote_Exception( 's3_auth', sprintf( __( 'The storage refused the access key (%s). Check the key, secret, region and bucket.', 'accessnow-site-replica' ), $error ) ); } if ( 'NoSuchBucket' === $error ) { throw new Remote_Exception( 's3_bucket', __( 'That bucket does not exist (check its name and region).', 'accessnow-site-replica' ) ); } if ( 'BadDigest' === $error || 'InvalidDigest' === $error ) { throw new Remote_Exception( 'upload_md5', __( 'A piece arrived damaged at the storage. It will be sent again.', 'accessnow-site-replica' ), true, 5 ); } /* translators: 1: HTTP status, 2: storage error code, 3: storage message. */ throw new Remote_Exception( 's3_http_' . $code, sprintf( __( 'The storage answered %1$d %2$s: %3$s', 'accessnow-site-replica' ), $code, $error, $message ) ); } /** * Signature Version 4 headers for a request. * * @param string $method Method. * @param string $url Address without the query. * @param array $query Query parameters. * @param array $headers Extra headers. * @param string $body Body. * @return array Headers to send (including Authorization). */ public function sign( $method, $url, array $query, array $headers, $body ) { $now = null === $this->now ? time() : (int) $this->now; $amzdate = gmdate( 'Ymd\THis\Z', $now ); $date = gmdate( 'Ymd', $now ); $parts = wp_parse_url( $url ); $host = $parts['host'] . ( isset( $parts['port'] ) ? ':' . $parts['port'] : '' ); $path = isset( $parts['path'] ) && '' !== $parts['path'] ? $parts['path'] : '/'; $hash = hash( 'sha256', $body ); $all = array( 'host' => $host, 'x-amz-content-sha256' => $hash, 'x-amz-date' => $amzdate, ); foreach ( $headers as $k => $v ) { $all[ strtolower( $k ) ] = trim( preg_replace( '/\s+/', ' ', (string) $v ) ); } ksort( $all ); $canonical_headers = ''; foreach ( $all as $k => $v ) { $canonical_headers .= $k . ':' . $v . "\n"; } $signed_headers = implode( ';', array_keys( $all ) ); $canonical = $method . "\n" . $path . "\n" . self::canonical_query( $query ) . "\n" . $canonical_headers . "\n" . $signed_headers . "\n" . $hash; $scope = $date . '/' . $this->region() . '/s3/aws4_request'; $to_sign = "AWS4-HMAC-SHA256\n" . $amzdate . "\n" . $scope . "\n" . hash( 'sha256', $canonical ); $k = hash_hmac( 'sha256', $date, 'AWS4' . $this->s['secret_key'], true ); $k = hash_hmac( 'sha256', $this->region(), $k, true ); $k = hash_hmac( 'sha256', 's3', $k, true ); $k = hash_hmac( 'sha256', 'aws4_request', $k, true ); $signature = hash_hmac( 'sha256', $to_sign, $k ); $out = array(); foreach ( $all as $k2 => $v ) { if ( 'host' !== $k2 ) { $out[ $k2 ] = $v; } } $out['Authorization'] = 'AWS4-HMAC-SHA256 Credential=' . $this->s['access_key'] . '/' . $scope . ', SignedHeaders=' . $signed_headers . ', Signature=' . $signature; return $out; } /** * Canonical query string (sorted, RFC 3986 encoded, "key=" for empty values). * * @param array $query Parameters. */ private static function canonical_query( array $query ) { ksort( $query, SORT_STRING ); $pairs = array(); foreach ( $query as $k => $v ) { $pairs[] = rawurlencode( (string) $k ) . '=' . rawurlencode( (string) $v ); } return implode( '&', $pairs ); } }