12345678module remote910import net.http11import time1213pub const s3_service = 's3'14pub const retry_attempts = 31516pub struct S3 {17pub:18 endpoint string19 bucket string20 region string21 access_key string22 secret_key string23mut:24 signer Signer25 root string 26}2728pub fn new_s3(endpoint string, bucket string, region string, access_key string, secret_key string) !S3 {29 if endpoint == '' || bucket == '' || region == '' {30 return error('the S3 remote needs an endpoint, a bucket and a region')31 }32 if access_key == '' || secret_key == '' {33 return error('the S3 remote needs an access key and a secret key')34 }35 if !endpoint.starts_with('http://') && !endpoint.starts_with('https://') {36 return error('the S3 endpoint should start with https:// — got "${endpoint}"')37 }38 root := endpoint.trim_right('/')39 return S3{40 endpoint: root41 bucket: bucket42 region: region43 access_key: access_key44 secret_key: secret_key45 signer: Signer{46 access_key: access_key47 secret_key: secret_key48 region: region49 service: s3_service50 sign_payload: true51 }52 root: root53 }54}5556pub fn (mut s S3) get(key string) !Object {57 response := s.send('GET', key, '', map[string]string{}, []u8{})!58 match response.status_code {59 404 {60 return Object{}61 }62 200 {63 return Object{64 found: true65 body: response.body.bytes()66 67 68 tag: header_of(response, 'ETag').trim_space()69 }70 }71 else {72 return error('s3 GET ${key} failed: ${response.status_code} ${short_body(response)}')73 }74 }75}7677pub fn (mut s S3) list(prefix string) ![]string {78 mut keys := []string{}79 mut token := ''80 for {81 82 83 mut query := 'list-type=2&prefix=${prefix}'84 if token != '' {85 query += '&continuation-token=${token}'86 }87 response := s.send('GET', '', query, map[string]string{}, []u8{})!88 if response.status_code != 200 {89 return error('s3 list failed: ${response.status_code} ${short_body(response)}')90 }91 keys << elements(response.body, 'Key')92 if !response.body.contains('<IsTruncated>true</IsTruncated>') {93 break94 }95 token = first_element(response.body, 'NextContinuationToken') or { break }96 }97 return keys98}99100pub fn (mut s S3) put_if_absent(key string, body []u8) !bool {101 response := s.send('PUT', key, '', {102 'If-None-Match': '*'103 }, body)!104 if response.status_code == 412 {105 return false106 }107 if !success(response.status_code) {108 return error('s3 PUT ${key} failed: ${response.status_code} ${short_body(response)}')109 }110 return true111}112113pub fn (mut s S3) put_if_match(key string, body []u8, tag string) !bool {114 mut headers := map[string]string{}115 if tag == '' {116 117 headers['If-None-Match'] = '*'118 } else {119 headers['If-Match'] = tag120 }121 response := s.send('PUT', key, '', headers, body)!122 if response.status_code == 412 {123 return false124 }125 if !success(response.status_code) {126 return error('s3 PUT ${key} failed: ${response.status_code} ${short_body(response)}')127 }128 return true129}130131pub fn (mut s S3) delete(key string) ! {132 response := s.send('DELETE', key, '', map[string]string{}, []u8{})!133 if !success(response.status_code) && response.status_code != 404 {134 return error('s3 DELETE ${key} failed: ${response.status_code} ${short_body(response)}')135 }136}137138139140fn (mut s S3) send(method string, key string, query string, headers map[string]string, body []u8) !http.Response {141 url := s.url_for(key, query)142 mut last := ''143 for attempt in 1 .. retry_attempts + 1 {144 signed := s.signer.sign(method, url, query, headers, body, time.now().local_to_utc())145 mut outbound := http.new_header()146 for name, value in signed {147 outbound.add_custom(header_case(name), value) or {}148 }149 response := http.fetch(150 url: url151 method: method_from(method)152 header: outbound153 data: body.bytestr()154 ) or {155 last = err.msg()156 time.sleep(backoff(attempt))157 continue158 }159 if response.status_code >= 500 {160 last = '${response.status_code}'161 time.sleep(backoff(attempt))162 continue163 }164 return response165 }166 return error('s3 ${method} ${key} failed after ${retry_attempts} attempts: ${last}')167}168169170171fn (s &S3) url_for(key string, query string) string {172 mut url := '${s.root}/${encode(s.bucket, false)}'173 if key != '' {174 url += '/${encode(key, true)}'175 }176 if query != '' {177 url += '?${canonical_query(query)}'178 }179 return url180}181182fn backoff(attempt int) time.Duration {183 return time.Duration(attempt) * 200 * time.millisecond184}185186fn method_from(method string) http.Method {187 return match method {188 'GET' { .get }189 'PUT' { .put }190 'POST' { .post }191 'DELETE' { .delete }192 'HEAD' { .head }193 else { .get }194 }195}196197fn success(status int) bool {198 return status >= 200 && status < 300199}200201fn header_of(response http.Response, name string) string {202 return response.header.get_custom(name, http.HeaderQueryConfig{}) or { '' }203}204205fn header_case(name string) string {206 return match name {207 'authorization' { 'Authorization' }208 'host' { 'Host' }209 'x-amz-date' { 'X-Amz-Date' }210 'x-amz-content-sha256' { 'X-Amz-Content-Sha256' }211 'if-none-match' { 'If-None-Match' }212 'if-match' { 'If-Match' }213 else { name }214 }215}216217fn short_body(response http.Response) string {218 body := response.body219 if body.len <= 200 {220 return body221 }222 return body[..200]223}224225226227228pub fn elements(xml string, name string) []string {229 open := '<${name}>'230 close := '</${name}>'231 mut out := []string{}232 mut rest := xml233 for {234 start := rest.index(open) or { break }235 after := rest[start + open.len..]236 end := after.index(close) or { break }237 out << unescape(after[..end])238 rest = after[end + close.len..]239 }240 return out241}242243fn first_element(xml string, name string) ?string {244 values := elements(xml, name)245 if values.len == 0 {246 return none247 }248 return values[0]249}250251fn unescape(text string) string {252 return text.replace('&', '&').replace('<', '<').replace('>', '>').replace('"',253 '"').replace(''', "'")254}