s3.v

1//! The S3 backend: plain HTTPS with SigV4 signing against any S3 endpoint.2//!3//! Only the two conditional writes matter for correctness — `If-None-Match: *`4//! for objects and saves, `If-Match` for `head` — because they are what stops5//! two machines from silently overwriting each other. Everything else is6//! ordinary GET, DELETE and ListObjectsV2.78module remote910import net.http11import time1213pub const s3_service = 's3'14pub const retry_attempts = 31516pub struct S3 {17pub:18	endpoint   string19	bucket     string20	region     string21	access_key string22	secret_key string23mut:24	signer Signer25	root   string // scheme://host[:port][/base path]26}2728pub fn new_s3(endpoint string, bucket string, region string, access_key string, secret_key string) !S3 {29	if endpoint == '' || bucket == '' || region == '' {30		return error('the S3 remote needs an endpoint, a bucket and a region')31	}32	if access_key == '' || secret_key == '' {33		return error('the S3 remote needs an access key and a secret key')34	}35	if !endpoint.starts_with('http://') && !endpoint.starts_with('https://') {36		return error('the S3 endpoint should start with https:// — got "${endpoint}"')37	}38	root := endpoint.trim_right('/')39	return S3{40		endpoint:   root41		bucket:     bucket42		region:     region43		access_key: access_key44		secret_key: secret_key45		signer:     Signer{46			access_key:   access_key47			secret_key:   secret_key48			region:       region49			service:      s3_service50			sign_payload: true51		}52		root:       root53	}54}5556pub fn (mut s S3) get(key string) !Object {57	response := s.send('GET', key, '', map[string]string{}, []u8{})!58	match response.status_code {59		404 {60			return Object{}61		}62		200 {63			return Object{64				found: true65				body:  response.body.bytes()66				tag:   clean_etag(header_of(response, 'ETag'))67			}68		}69		else {70			return error('s3 GET ${key} failed: ${response.status_code} ${short_body(response)}')71		}72	}73}7475pub fn (mut s S3) list(prefix string) ![]string {76	mut keys := []string{}77	mut token := ''78	for {79		// The query is passed raw; url_for and the signer encode it the same80		// way, which is what keeps the URL and the signature in step.81		mut query := 'list-type=2&prefix=${prefix}'82		if token != '' {83			query += '&continuation-token=${token}'84		}85		response := s.send('GET', '', query, map[string]string{}, []u8{})!86		if response.status_code != 200 {87			return error('s3 list failed: ${response.status_code} ${short_body(response)}')88		}89		keys << elements(response.body, 'Key')90		if !response.body.contains('<IsTruncated>true</IsTruncated>') {91			break92		}93		token = first_element(response.body, 'NextContinuationToken') or { break }94	}95	return keys96}9798pub fn (mut s S3) put_if_absent(key string, body []u8) !bool {99	response := s.send('PUT', key, '', {100		'If-None-Match': '*'101	}, body)!102	if response.status_code == 412 {103		return false104	}105	if !success(response.status_code) {106		return error('s3 PUT ${key} failed: ${response.status_code} ${short_body(response)}')107	}108	return true109}110111pub fn (mut s S3) put_if_match(key string, body []u8, tag string) !bool {112	mut headers := map[string]string{}113	if tag == '' {114		// The key must not exist yet.115		headers['If-None-Match'] = '*'116	} else {117		headers['If-Match'] = tag118	}119	response := s.send('PUT', key, '', headers, body)!120	if response.status_code == 412 {121		return false122	}123	if !success(response.status_code) {124		return error('s3 PUT ${key} failed: ${response.status_code} ${short_body(response)}')125	}126	return true127}128129pub fn (mut s S3) delete(key string) ! {130	response := s.send('DELETE', key, '', map[string]string{}, []u8{})!131	if !success(response.status_code) && response.status_code != 404 {132		return error('s3 DELETE ${key} failed: ${response.status_code} ${short_body(response)}')133	}134}135136// send signs one request and retries the failures worth retrying: a network137// error, or a 5xx. A 412 is an answer, not a fault, and is never retried.138fn (mut s S3) send(method string, key string, query string, headers map[string]string, body []u8) !http.Response {139	url := s.url_for(key, query)140	mut last := ''141	for attempt in 1 .. retry_attempts + 1 {142		signed := s.signer.sign(method, url, query, headers, body, time.now().local_to_utc())143		mut outbound := http.new_header()144		for name, value in signed {145			outbound.add_custom(header_case(name), value) or {}146		}147		response := http.fetch(148			url:    url149			method: method_from(method)150			header: outbound151			data:   body.bytestr()152		) or {153			last = err.msg()154			time.sleep(backoff(attempt))155			continue156		}157		if response.status_code >= 500 {158			last = '${response.status_code}'159			time.sleep(backoff(attempt))160			continue161		}162		return response163	}164	return error('s3 ${method} ${key} failed after ${retry_attempts} attempts: ${last}')165}166167// url_for builds the path-style URL. The path is encoded here exactly as the168// signer encodes it for the canonical request.169fn (s &S3) url_for(key string, query string) string {170	mut url := '${s.root}/${encode(s.bucket, false)}'171	if key != '' {172		url += '/${encode(key, true)}'173	}174	if query != '' {175		url += '?${canonical_query(query)}'176	}177	return url178}179180fn backoff(attempt int) time.Duration {181	return time.Duration(attempt) * 200 * time.millisecond182}183184fn method_from(method string) http.Method {185	return match method {186		'GET' { .get }187		'PUT' { .put }188		'POST' { .post }189		'DELETE' { .delete }190		'HEAD' { .head }191		else { .get }192	}193}194195fn success(status int) bool {196	return status >= 200 && status < 300197}198199fn header_of(response http.Response, name string) string {200	return response.header.get_custom(name, http.HeaderQueryConfig{}) or { '' }201}202203// clean_etag strips the quotes S3 puts around an ETag: the value is sent back204// verbatim in `If-Match`, and a quoted form would not match.205pub fn clean_etag(value string) string {206	return value.trim_space().trim('"')207}208209fn header_case(name string) string {210	return match name {211		'authorization' { 'Authorization' }212		'host' { 'Host' }213		'x-amz-date' { 'X-Amz-Date' }214		'x-amz-content-sha256' { 'X-Amz-Content-Sha256' }215		'if-none-match' { 'If-None-Match' }216		'if-match' { 'If-Match' }217		else { name }218	}219}220221fn short_body(response http.Response) string {222	body := response.body223	if body.len <= 200 {224		return body225	}226	return body[..200]227}228229// elements pulls every text between <name> and </name> out of a response. The230// only response Koh reads is S3's ListObjectsV2, which is flat and small, so a231// scanner is honest here where a full XML parser would be a dependency.232pub fn elements(xml string, name string) []string {233	open := '<${name}>'234	close := '</${name}>'235	mut out := []string{}236	mut rest := xml237	for {238		start := rest.index(open) or { break }239		after := rest[start + open.len..]240		end := after.index(close) or { break }241		out << unescape(after[..end])242		rest = after[end + close.len..]243	}244	return out245}246247fn first_element(xml string, name string) ?string {248	values := elements(xml, name)249	if values.len == 0 {250		return none251	}252	return values[0]253}254255fn unescape(text string) string {256	return text.replace('&amp;', '&').replace('&lt;', '<').replace('&gt;', '>').replace('&quot;',257		'"').replace('&#39;', "'")258}