s3.v

1//! The S3 backend: plain HTTPS with SigV4 signing against any S3 endpoint.2//!3//! Only the two conditional writes matter for correctness — `If-None-Match: *`4//! for objects and saves, `If-Match` for `head` — because they are what stops5//! two machines from silently overwriting each other. Everything else is6//! ordinary GET, DELETE and ListObjectsV2.78module remote910import net.http11import time1213pub const s3_service = 's3'14pub const retry_attempts = 31516pub struct S3 {17pub:18	endpoint   string19	bucket     string20	region     string21	access_key string22	secret_key string23mut:24	signer Signer25	root   string // scheme://host[:port][/base path]26}2728pub fn new_s3(endpoint string, bucket string, region string, access_key string, secret_key string) !S3 {29	if endpoint == '' || bucket == '' || region == '' {30		return error('the S3 remote needs an endpoint, a bucket and a region')31	}32	if access_key == '' || secret_key == '' {33		return error('the S3 remote needs an access key and a secret key')34	}35	if !endpoint.starts_with('http://') && !endpoint.starts_with('https://') {36		return error('the S3 endpoint should start with https:// — got "${endpoint}"')37	}38	root := endpoint.trim_right('/')39	return S3{40		endpoint:   root41		bucket:     bucket42		region:     region43		access_key: access_key44		secret_key: secret_key45		signer:     Signer{46			access_key:   access_key47			secret_key:   secret_key48			region:       region49			service:      s3_service50			sign_payload: true51		}52		root:       root53	}54}5556// get answers 404 as not found, not an error. The ETag comes back as `tag` for put_if_match.57pub fn (mut s S3) get(key string) !Object {58	response := s.send('GET', key, '', map[string]string{}, []u8{})!59	match response.status_code {60		404 {61			return Object{}62		}63		200 {64			return Object{65				found: true66				body:  response.body.bytes()67				// Sent back as-is in If-Match: an RFC 7232 server compares the68				// quoted entity-tag, and a lenient one strips the quotes itself.69				tag:   header_of(response, 'ETag').trim_space()70			}71		}72		else {73			return error('s3 GET ${key} failed: ${response.status_code} ${short_body(response)}')74		}75	}76}7778// list follows ListObjectsV2 continuation tokens until the listing is no longer truncated.79pub fn (mut s S3) list(prefix string) ![]string {80	mut keys := []string{}81	mut token := ''82	for {83		// The query is passed raw; url_for and the signer encode it the same84		// way, which is what keeps the URL and the signature in step.85		mut query := 'list-type=2&prefix=${prefix}'86		if token != '' {87			query += '&continuation-token=${token}'88		}89		response := s.send('GET', '', query, map[string]string{}, []u8{})!90		if response.status_code != 200 {91			return error('s3 list failed: ${response.status_code} ${short_body(response)}')92		}93		keys << elements(response.body, 'Key')94		if !response.body.contains('<IsTruncated>true</IsTruncated>') {95			break96		}97		token = first_element(response.body, 'NextContinuationToken') or { break }98	}99	return keys100}101102// put_if_absent sends If-None-Match: *. A 412 means the key exists and nothing was written: false, not an error.103pub fn (mut s S3) put_if_absent(key string, body []u8) !bool {104	response := s.send('PUT', key, '', {105		'If-None-Match': '*'106	}, body)!107	if response.status_code == 412 {108		return false109	}110	if !success(response.status_code) {111		return error('s3 PUT ${key} failed: ${response.status_code} ${short_body(response)}')112	}113	return true114}115116// put_if_match is the only overwrite: If-Match with the tag from get, or If-None-Match: * when there was no117// head yet (tag ). A 412 means another machine moved the key first: false, not an error.118pub fn (mut s S3) put_if_match(key string, body []u8, tag string) !bool {119	mut headers := map[string]string{}120	if tag == '' {121		// The key must not exist yet.122		headers['If-None-Match'] = '*'123	} else {124		headers['If-Match'] = tag125	}126	response := s.send('PUT', key, '', headers, body)!127	if response.status_code == 412 {128		return false129	}130	if !success(response.status_code) {131		return error('s3 PUT ${key} failed: ${response.status_code} ${short_body(response)}')132	}133	return true134}135136// delete counts a missing key as done.137pub fn (mut s S3) delete(key string) ! {138	response := s.send('DELETE', key, '', map[string]string{}, []u8{})!139	if !success(response.status_code) && response.status_code != 404 {140		return error('s3 DELETE ${key} failed: ${response.status_code} ${short_body(response)}')141	}142}143144// send signs one request and retries the failures worth retrying: a network145// error, or a 5xx. A 412 is an answer, not a fault, and is never retried.146fn (mut s S3) send(method string, key string, query string, headers map[string]string, body []u8) !http.Response {147	url := s.url_for(key, query)148	mut last := ''149	for attempt in 1 .. retry_attempts + 1 {150		signed := s.signer.sign(method, url, query, headers, body, time.now().local_to_utc())151		mut outbound := http.new_header()152		for name, value in signed {153			outbound.add_custom(header_case(name), value) or {}154		}155		// V's HTTP/2 client gives a whole upload five minutes, then fails it,156		// so a big object on a slow uplink could never be offered. Each157		// request opens its own connection anyway, so HTTP/2 gains nothing.158		response := http.fetch(159			url:          url160			method:       method_from(method)161			header:       outbound162			data:         body.bytestr()163			enable_http2: false164		) or {165			last = err.msg()166			time.sleep(backoff(attempt))167			continue168		}169		if response.status_code >= 500 {170			last = '${response.status_code}'171			time.sleep(backoff(attempt))172			continue173		}174		return response175	}176	return error('s3 ${method} ${key} failed after ${retry_attempts} attempts: ${last}')177}178179// url_for builds the path-style URL. The path is encoded here exactly as the180// signer encodes it for the canonical request.181fn (s &S3) url_for(key string, query string) string {182	mut url := '${s.root}/${encode(s.bucket, false)}'183	if key != '' {184		url += '/${encode(key, true)}'185	}186	if query != '' {187		url += '?${canonical_query(query)}'188	}189	return url190}191192fn backoff(attempt int) time.Duration {193	return time.Duration(attempt) * 200 * time.millisecond194}195196fn method_from(method string) http.Method {197	return match method {198		'GET' { .get }199		'PUT' { .put }200		'POST' { .post }201		'DELETE' { .delete }202		'HEAD' { .head }203		else { .get }204	}205}206207fn success(status int) bool {208	return status >= 200 && status < 300209}210211fn header_of(response http.Response, name string) string {212	return response.header.get_custom(name, http.HeaderQueryConfig{}) or { '' }213}214215fn header_case(name string) string {216	return match name {217		'authorization' { 'Authorization' }218		'host' { 'Host' }219		'x-amz-date' { 'X-Amz-Date' }220		'x-amz-content-sha256' { 'X-Amz-Content-Sha256' }221		'if-none-match' { 'If-None-Match' }222		'if-match' { 'If-Match' }223		else { name }224	}225}226227fn short_body(response http.Response) string {228	body := response.body229	if body.len <= 200 {230		return body231	}232	return body[..200]233}234235// elements takes every text between <name> and </name> out of a response. The236// only response Koh reads is S3's ListObjectsV2, which is flat and small, so a237// scanner is honest here where a full XML parser would be a dependency.238pub fn elements(xml string, name string) []string {239	open := '<${name}>'240	close := '</${name}>'241	mut out := []string{}242	mut rest := xml243	for {244		start := rest.index(open) or { break }245		after := rest[start + open.len..]246		end := after.index(close) or { break }247		out << unescape(after[..end])248		rest = after[end + close.len..]249	}250	return out251}252253fn first_element(xml string, name string) ?string {254	values := elements(xml, name)255	if values.len == 0 {256		return none257	}258	return values[0]259}260261fn unescape(text string) string {262	return text.replace('&amp;', '&').replace('&lt;', '<').replace('&gt;', '>').replace('&quot;',263		'"').replace('&#39;', "'")264}