s3.v

1//! The S3 backend: plain HTTPS with SigV4 signing against any S3 endpoint.2//!3//! Only the two conditional writes matter for correctness — `If-None-Match: *`4//! for objects and saves, `If-Match` for `head` — because they are what stops5//! two machines from silently overwriting each other. Everything else is6//! ordinary GET, DELETE and ListObjectsV2.78module remote910import net.http11import time1213pub const s3_service = 's3'14pub const retry_attempts = 31516pub struct S3 {17pub:18	endpoint   string19	bucket     string20	region     string21	access_key string22	secret_key string23mut:24	signer Signer25	root   string // scheme://host[:port][/base path]26}2728pub fn new_s3(endpoint string, bucket string, region string, access_key string, secret_key string) !S3 {29	if endpoint == '' || bucket == '' || region == '' {30		return error('the S3 remote needs an endpoint, a bucket and a region')31	}32	if access_key == '' || secret_key == '' {33		return error('the S3 remote needs an access key and a secret key')34	}35	if !endpoint.starts_with('http://') && !endpoint.starts_with('https://') {36		return error('the S3 endpoint should start with https:// — got "${endpoint}"')37	}38	root := endpoint.trim_right('/')39	return S3{40		endpoint:   root41		bucket:     bucket42		region:     region43		access_key: access_key44		secret_key: secret_key45		signer:     Signer{46			access_key:   access_key47			secret_key:   secret_key48			region:       region49			service:      s3_service50			sign_payload: true51		}52		root:       root53	}54}5556// get answers 404 as not found, not an error. The ETag comes back as `tag` for put_if_match.57pub fn (mut s S3) get(key string) !Object {58	response := s.send('GET', key, '', map[string]string{}, []u8{})!59	match response.status_code {60		404 {61			return Object{}62		}63		200 {64			return Object{65				found: true66				body:  response.body.bytes()67				// Sent back as-is in If-Match: an RFC 7232 server compares the68				// quoted entity-tag, and a lenient one strips the quotes itself.69				tag:   header_of(response, 'ETag').trim_space()70			}71		}72		else {73			return error('s3 GET ${key} failed: ${response.status_code} ${short_body(response)}')74		}75	}76}7778// list follows ListObjectsV2 continuation tokens until the listing is no longer truncated, or a79// truncated one names no next token.80pub fn (mut s S3) list(prefix string) ![]string {81	mut keys := []string{}82	mut token := ''83	for {84		// The query is passed raw; url_for and the signer encode it the same85		// way, which is what keeps the URL and the signature in step.86		mut query := 'list-type=2&prefix=${prefix}'87		if token != '' {88			query += '&continuation-token=${token}'89		}90		response := s.send('GET', '', query, map[string]string{}, []u8{})!91		if response.status_code != 200 {92			return error('s3 list failed: ${response.status_code} ${short_body(response)}')93		}94		keys << elements(response.body, 'Key')95		if !response.body.contains('<IsTruncated>true</IsTruncated>') {96			break97		}98		token = first_element(response.body, 'NextContinuationToken') or { break }99	}100	return keys101}102103// put_if_absent sends If-None-Match: *. A 412 means the key exists and nothing was written: false, not an error.104pub fn (mut s S3) put_if_absent(key string, body []u8) !bool {105	response := s.send('PUT', key, '', {106		'If-None-Match': '*'107	}, body)!108	if response.status_code == 412 {109		return false110	}111	if !success(response.status_code) {112		return error('s3 PUT ${key} failed: ${response.status_code} ${short_body(response)}')113	}114	return true115}116117// put_if_match is the only overwrite: If-Match with the tag from get, or If-None-Match: * when there was no118// head yet (tag ). A 412 means another machine moved the key first: false, not an error.119pub fn (mut s S3) put_if_match(key string, body []u8, tag string) !bool {120	mut headers := map[string]string{}121	if tag == '' {122		// The key must not exist yet.123		headers['If-None-Match'] = '*'124	} else {125		headers['If-Match'] = tag126	}127	response := s.send('PUT', key, '', headers, body)!128	if response.status_code == 412 {129		return false130	}131	if !success(response.status_code) {132		return error('s3 PUT ${key} failed: ${response.status_code} ${short_body(response)}')133	}134	return true135}136137// delete counts a missing key as done.138pub fn (mut s S3) delete(key string) ! {139	response := s.send('DELETE', key, '', map[string]string{}, []u8{})!140	if !success(response.status_code) && response.status_code != 404 {141		return error('s3 DELETE ${key} failed: ${response.status_code} ${short_body(response)}')142	}143}144145// send signs one request and retries the failures worth retrying: a network146// error, or a 5xx. A 412 is an answer, not a fault, and is never retried.147fn (mut s S3) send(method string, key string, query string, headers map[string]string, body []u8) !http.Response {148	url := s.url_for(key, query)149	mut last := ''150	for attempt in 1 .. retry_attempts + 1 {151		signed := s.signer.sign(method, url, query, headers, body, time.now().local_to_utc())152		mut outbound := http.new_header()153		for name, value in signed {154			outbound.add_custom(header_case(name), value) or {}155		}156		// V's HTTP/2 client gives a whole upload five minutes, then fails it,157		// so a big object on a slow uplink could never be offered. Each158		// request opens its own connection anyway, so HTTP/2 gains nothing.159		response := http.fetch(160			url:          url161			validate:     true162			verify:       tls_roots(url)!163			method:       method_from(method)164			header:       outbound165			data:         body.bytestr()166			enable_http2: false167		) or {168			last = err.msg()169			time.sleep(backoff(attempt))170			continue171		}172		if response.status_code >= 500 {173			last = '${response.status_code}'174			time.sleep(backoff(attempt))175			continue176		}177		return response178	}179	return error('s3 ${method} ${key} failed after ${retry_attempts} attempts: ${last}')180}181182// url_for builds the path-style URL. The path is encoded here exactly as the183// signer encodes it for the canonical request.184fn (s &S3) url_for(key string, query string) string {185	mut url := '${s.root}/${encode(s.bucket, false)}'186	if key != '' {187		url += '/${encode(key, true)}'188	}189	if query != '' {190		url += '?${canonical_query(query)}'191	}192	return url193}194195fn backoff(attempt int) time.Duration {196	return time.Duration(attempt) * 200 * time.millisecond197}198199fn method_from(method string) http.Method {200	return match method {201		'GET' { .get }202		'PUT' { .put }203		'POST' { .post }204		'DELETE' { .delete }205		'HEAD' { .head }206		else { .get }207	}208}209210fn success(status int) bool {211	return status >= 200 && status < 300212}213214fn header_of(response http.Response, name string) string {215	return response.header.get_custom(name, http.HeaderQueryConfig{}) or { '' }216}217218fn header_case(name string) string {219	return match name {220		'authorization' { 'Authorization' }221		'host' { 'Host' }222		'x-amz-date' { 'X-Amz-Date' }223		'x-amz-content-sha256' { 'X-Amz-Content-Sha256' }224		'if-none-match' { 'If-None-Match' }225		'if-match' { 'If-Match' }226		else { name }227	}228}229230fn short_body(response http.Response) string {231	body := response.body232	if body.len <= 200 {233		return body234	}235	return body[..200]236}237238// elements takes every text between <name> and </name> out of a response. The239// only response Koh reads is S3's ListObjectsV2, which is flat and small, so a240// scanner is honest here where a full XML parser would be a dependency.241pub fn elements(xml string, name string) []string {242	open := '<${name}>'243	close := '</${name}>'244	mut out := []string{}245	mut rest := xml246	for {247		start := rest.index(open) or { break }248		after := rest[start + open.len..]249		end := after.index(close) or { break }250		out << unescape(after[..end])251		rest = after[end + close.len..]252	}253	return out254}255256fn first_element(xml string, name string) ?string {257	values := elements(xml, name)258	if values.len == 0 {259		return none260	}261	return values[0]262}263264fn unescape(text string) string {265	return text.replace('&amp;', '&').replace('&lt;', '<').replace('&gt;', '>').replace('&quot;',266		'"').replace('&#39;', "'")267}