s3.v

1//! The S3 backend: plain HTTPS with SigV4 signing against any S3 endpoint.2//!3//! Only the two conditional writes matter for correctness — `If-None-Match: *`4//! for objects and saves, `If-Match` for `head` — because they are what stops5//! two machines from silently overwriting each other. Everything else is6//! ordinary GET, DELETE and ListObjectsV2.78module remote910import net.http11import time1213pub const s3_service = 's3'14pub const retry_attempts = 31516pub struct S3 {17pub:18	endpoint   string19	bucket     string20	region     string21	access_key string22	secret_key string23mut:24	signer Signer25	root   string // scheme://host[:port][/base path]26}2728pub fn new_s3(endpoint string, bucket string, region string, access_key string, secret_key string) !S3 {29	if endpoint == '' || bucket == '' || region == '' {30		return error('the S3 remote needs an endpoint, a bucket and a region')31	}32	if access_key == '' || secret_key == '' {33		return error('the S3 remote needs an access key and a secret key')34	}35	if !endpoint.starts_with('http://') && !endpoint.starts_with('https://') {36		return error('the S3 endpoint should start with https:// — got "${endpoint}"')37	}38	root := endpoint.trim_right('/')39	return S3{40		endpoint:   root41		bucket:     bucket42		region:     region43		access_key: access_key44		secret_key: secret_key45		signer:     Signer{46			access_key:   access_key47			secret_key:   secret_key48			region:       region49			service:      s3_service50			sign_payload: true51		}52		root:       root53	}54}5556pub fn (mut s S3) get(key string) !Object {57	response := s.send('GET', key, '', map[string]string{}, []u8{})!58	match response.status_code {59		404 {60			return Object{}61		}62		200 {63			return Object{64				found: true65				body:  response.body.bytes()66				// Sent back as-is in If-Match: an RFC 7232 server compares the67				// quoted entity-tag, and a lenient one strips the quotes itself.68				tag:   header_of(response, 'ETag').trim_space()69			}70		}71		else {72			return error('s3 GET ${key} failed: ${response.status_code} ${short_body(response)}')73		}74	}75}7677pub fn (mut s S3) list(prefix string) ![]string {78	mut keys := []string{}79	mut token := ''80	for {81		// The query is passed raw; url_for and the signer encode it the same82		// way, which is what keeps the URL and the signature in step.83		mut query := 'list-type=2&prefix=${prefix}'84		if token != '' {85			query += '&continuation-token=${token}'86		}87		response := s.send('GET', '', query, map[string]string{}, []u8{})!88		if response.status_code != 200 {89			return error('s3 list failed: ${response.status_code} ${short_body(response)}')90		}91		keys << elements(response.body, 'Key')92		if !response.body.contains('<IsTruncated>true</IsTruncated>') {93			break94		}95		token = first_element(response.body, 'NextContinuationToken') or { break }96	}97	return keys98}99100pub fn (mut s S3) put_if_absent(key string, body []u8) !bool {101	response := s.send('PUT', key, '', {102		'If-None-Match': '*'103	}, body)!104	if response.status_code == 412 {105		return false106	}107	if !success(response.status_code) {108		return error('s3 PUT ${key} failed: ${response.status_code} ${short_body(response)}')109	}110	return true111}112113pub fn (mut s S3) put_if_match(key string, body []u8, tag string) !bool {114	mut headers := map[string]string{}115	if tag == '' {116		// The key must not exist yet.117		headers['If-None-Match'] = '*'118	} else {119		headers['If-Match'] = tag120	}121	response := s.send('PUT', key, '', headers, body)!122	if response.status_code == 412 {123		return false124	}125	if !success(response.status_code) {126		return error('s3 PUT ${key} failed: ${response.status_code} ${short_body(response)}')127	}128	return true129}130131pub fn (mut s S3) delete(key string) ! {132	response := s.send('DELETE', key, '', map[string]string{}, []u8{})!133	if !success(response.status_code) && response.status_code != 404 {134		return error('s3 DELETE ${key} failed: ${response.status_code} ${short_body(response)}')135	}136}137138// send signs one request and retries the failures worth retrying: a network139// error, or a 5xx. A 412 is an answer, not a fault, and is never retried.140fn (mut s S3) send(method string, key string, query string, headers map[string]string, body []u8) !http.Response {141	url := s.url_for(key, query)142	mut last := ''143	for attempt in 1 .. retry_attempts + 1 {144		signed := s.signer.sign(method, url, query, headers, body, time.now().local_to_utc())145		mut outbound := http.new_header()146		for name, value in signed {147			outbound.add_custom(header_case(name), value) or {}148		}149		// V's HTTP/2 client gives a whole upload five minutes, then fails it,150		// so a big object on a slow uplink could never be offered. Each151		// request opens its own connection anyway, so HTTP/2 gains nothing.152		response := http.fetch(153			url:          url154			method:       method_from(method)155			header:       outbound156			data:         body.bytestr()157			enable_http2: false158		) or {159			last = err.msg()160			time.sleep(backoff(attempt))161			continue162		}163		if response.status_code >= 500 {164			last = '${response.status_code}'165			time.sleep(backoff(attempt))166			continue167		}168		return response169	}170	return error('s3 ${method} ${key} failed after ${retry_attempts} attempts: ${last}')171}172173// url_for builds the path-style URL. The path is encoded here exactly as the174// signer encodes it for the canonical request.175fn (s &S3) url_for(key string, query string) string {176	mut url := '${s.root}/${encode(s.bucket, false)}'177	if key != '' {178		url += '/${encode(key, true)}'179	}180	if query != '' {181		url += '?${canonical_query(query)}'182	}183	return url184}185186fn backoff(attempt int) time.Duration {187	return time.Duration(attempt) * 200 * time.millisecond188}189190fn method_from(method string) http.Method {191	return match method {192		'GET' { .get }193		'PUT' { .put }194		'POST' { .post }195		'DELETE' { .delete }196		'HEAD' { .head }197		else { .get }198	}199}200201fn success(status int) bool {202	return status >= 200 && status < 300203}204205fn header_of(response http.Response, name string) string {206	return response.header.get_custom(name, http.HeaderQueryConfig{}) or { '' }207}208209fn header_case(name string) string {210	return match name {211		'authorization' { 'Authorization' }212		'host' { 'Host' }213		'x-amz-date' { 'X-Amz-Date' }214		'x-amz-content-sha256' { 'X-Amz-Content-Sha256' }215		'if-none-match' { 'If-None-Match' }216		'if-match' { 'If-Match' }217		else { name }218	}219}220221fn short_body(response http.Response) string {222	body := response.body223	if body.len <= 200 {224		return body225	}226	return body[..200]227}228229// elements takes every text between <name> and </name> out of a response. The230// only response Koh reads is S3's ListObjectsV2, which is flat and small, so a231// scanner is honest here where a full XML parser would be a dependency.232pub fn elements(xml string, name string) []string {233	open := '<${name}>'234	close := '</${name}>'235	mut out := []string{}236	mut rest := xml237	for {238		start := rest.index(open) or { break }239		after := rest[start + open.len..]240		end := after.index(close) or { break }241		out << unescape(after[..end])242		rest = after[end + close.len..]243	}244	return out245}246247fn first_element(xml string, name string) ?string {248	values := elements(xml, name)249	if values.len == 0 {250		return none251	}252	return values[0]253}254255fn unescape(text string) string {256	return text.replace('&amp;', '&').replace('&lt;', '<').replace('&gt;', '>').replace('&quot;',257		'"').replace('&#39;', "'")258}