s3.v

1//! The S3 backend: plain HTTPS with SigV4 signing against any S3 endpoint.2//!3//! Only the two conditional writes matter for correctness — `If-None-Match: *`4//! for objects and saves, `If-Match` for `head` — because they are what stops5//! two machines from silently overwriting each other. Everything else is6//! ordinary GET, DELETE and ListObjectsV2.78module remote910import net.http11import time1213pub const s3_service = 's3'14pub const retry_attempts = 31516pub struct S3 {17pub:18	endpoint   string19	bucket     string20	region     string21	access_key string22	secret_key string23mut:24	signer Signer25	root   string // scheme://host[:port][/base path]26}2728pub fn new_s3(endpoint string, bucket string, region string, access_key string, secret_key string) !S3 {29	if endpoint == '' || bucket == '' || region == '' {30		return error('the S3 remote needs an endpoint, a bucket and a region')31	}32	if access_key == '' || secret_key == '' {33		return error('the S3 remote needs an access key and a secret key')34	}35	if !endpoint.starts_with('http://') && !endpoint.starts_with('https://') {36		return error('the S3 endpoint should start with https:// — got "${endpoint}"')37	}38	root := endpoint.trim_right('/')39	return S3{40		endpoint:   root41		bucket:     bucket42		region:     region43		access_key: access_key44		secret_key: secret_key45		signer:     Signer{46			access_key:   access_key47			secret_key:   secret_key48			region:       region49			service:      s3_service50			sign_payload: true51		}52		root:       root53	}54}5556// get answers 404 as not found, not an error. The ETag comes back as `tag` for put_if_match.57pub fn (mut s S3) get(key string) !Object {58	response := s.send('GET', key, '', map[string]string{}, []u8{})!59	match response.status_code {60		404 {61			return Object{}62		}63		200 {64			return Object{65				found: true66				body:  response.body.bytes()67				// Sent back as-is in If-Match: an RFC 7232 server compares the68				// quoted entity-tag, and a lenient one strips the quotes itself.69				tag:   header_of(response, 'ETag').trim_space()70			}71		}72		else {73			return error('s3 GET ${key} failed: ${response.status_code} ${short_body(response)}')74		}75	}76}7778// list follows ListObjectsV2 continuation tokens until the listing is no longer truncated, or a79// truncated one names no next token.80pub fn (mut s S3) list(prefix string) ![]string {81	mut keys := []string{}82	mut token := ''83	for {84		// The query is passed raw; url_for and the signer encode it the same85		// way, which is what keeps the URL and the signature in step.86		mut query := 'list-type=2&prefix=${prefix}'87		if token != '' {88			query += '&continuation-token=${token}'89		}90		response := s.send('GET', '', query, map[string]string{}, []u8{})!91		if response.status_code != 200 {92			return error('s3 list failed: ${response.status_code} ${short_body(response)}')93		}94		keys << elements(response.body, 'Key')95		if !response.body.contains('<IsTruncated>true</IsTruncated>') {96			break97		}98		token = first_element(response.body, 'NextContinuationToken') or { break }99	}100	return keys101}102103// put_if_absent sends If-None-Match: *. A 412 means the key exists and nothing was written: false, not an error.104pub fn (mut s S3) put_if_absent(key string, body []u8) !bool {105	response := s.send('PUT', key, '', {106		'If-None-Match': '*'107	}, body)!108	if response.status_code == 412 {109		return false110	}111	if !success(response.status_code) {112		return error('s3 PUT ${key} failed: ${response.status_code} ${short_body(response)}')113	}114	return true115}116117// put_if_match is the only overwrite: If-Match with the tag from get, or If-None-Match: * when there was no118// head yet (tag ). A 412 means another machine moved the key first: false, not an error.119pub fn (mut s S3) put_if_match(key string, body []u8, tag string) !bool {120	mut headers := map[string]string{}121	if tag == '' {122		// The key must not exist yet.123		headers['If-None-Match'] = '*'124	} else {125		headers['If-Match'] = tag126	}127	response := s.send('PUT', key, '', headers, body)!128	if response.status_code == 412 {129		return false130	}131	if !success(response.status_code) {132		return error('s3 PUT ${key} failed: ${response.status_code} ${short_body(response)}')133	}134	return true135}136137// delete counts a missing key as done.138pub fn (mut s S3) delete(key string) ! {139	response := s.send('DELETE', key, '', map[string]string{}, []u8{})!140	if !success(response.status_code) && response.status_code != 404 {141		return error('s3 DELETE ${key} failed: ${response.status_code} ${short_body(response)}')142	}143}144145// send signs one request and retries the failures worth retrying: a network146// error, or a 5xx. A 412 is an answer, not a fault, and is never retried.147fn (mut s S3) send(method string, key string, query string, headers map[string]string, body []u8) !http.Response {148	url := s.url_for(key, query)149	mut last := ''150	for attempt in 1 .. retry_attempts + 1 {151		signed := s.signer.sign(method, url, query, headers, body, time.now().local_to_utc())152		mut outbound := http.new_header()153		for name, value in signed {154			outbound.add_custom(header_case(name), value) or {}155		}156		// V's HTTP/2 client gives a whole upload five minutes, then fails it,157		// so a big object on a slow uplink could never be offered. Each158		// request opens its own connection anyway, so HTTP/2 gains nothing.159		response := http.fetch(160			url:          url161			method:       method_from(method)162			header:       outbound163			data:         body.bytestr()164			enable_http2: false165		) or {166			last = err.msg()167			time.sleep(backoff(attempt))168			continue169		}170		if response.status_code >= 500 {171			last = '${response.status_code}'172			time.sleep(backoff(attempt))173			continue174		}175		return response176	}177	return error('s3 ${method} ${key} failed after ${retry_attempts} attempts: ${last}')178}179180// url_for builds the path-style URL. The path is encoded here exactly as the181// signer encodes it for the canonical request.182fn (s &S3) url_for(key string, query string) string {183	mut url := '${s.root}/${encode(s.bucket, false)}'184	if key != '' {185		url += '/${encode(key, true)}'186	}187	if query != '' {188		url += '?${canonical_query(query)}'189	}190	return url191}192193fn backoff(attempt int) time.Duration {194	return time.Duration(attempt) * 200 * time.millisecond195}196197fn method_from(method string) http.Method {198	return match method {199		'GET' { .get }200		'PUT' { .put }201		'POST' { .post }202		'DELETE' { .delete }203		'HEAD' { .head }204		else { .get }205	}206}207208fn success(status int) bool {209	return status >= 200 && status < 300210}211212fn header_of(response http.Response, name string) string {213	return response.header.get_custom(name, http.HeaderQueryConfig{}) or { '' }214}215216fn header_case(name string) string {217	return match name {218		'authorization' { 'Authorization' }219		'host' { 'Host' }220		'x-amz-date' { 'X-Amz-Date' }221		'x-amz-content-sha256' { 'X-Amz-Content-Sha256' }222		'if-none-match' { 'If-None-Match' }223		'if-match' { 'If-Match' }224		else { name }225	}226}227228fn short_body(response http.Response) string {229	body := response.body230	if body.len <= 200 {231		return body232	}233	return body[..200]234}235236// elements takes every text between <name> and </name> out of a response. The237// only response Koh reads is S3's ListObjectsV2, which is flat and small, so a238// scanner is honest here where a full XML parser would be a dependency.239pub fn elements(xml string, name string) []string {240	open := '<${name}>'241	close := '</${name}>'242	mut out := []string{}243	mut rest := xml244	for {245		start := rest.index(open) or { break }246		after := rest[start + open.len..]247		end := after.index(close) or { break }248		out << unescape(after[..end])249		rest = after[end + close.len..]250	}251	return out252}253254fn first_element(xml string, name string) ?string {255	values := elements(xml, name)256	if values.len == 0 {257		return none258	}259	return values[0]260}261262fn unescape(text string) string {263	return text.replace('&amp;', '&').replace('&lt;', '<').replace('&gt;', '>').replace('&quot;',264		'"').replace('&#39;', "'")265}