//! The web backend: read-only access to a public Koh mirror. module remote import net.http import net.urllib pub struct Web { pub mut: base string // on_body hears how much of a response has arrived, each time it passes another MiB. on_body fn (received u64) = unsafe { nil } } pub fn new_web(base string) !Web { return Web{ base: kepr_host(base)! } } // kepr_host normalises a Kepr address: https unless the scheme says otherwise, and scheme, host // and port only. A path, a query, a fragment or credentials would send the code somewhere else. pub fn kepr_host(value string) !string { for letter in value { if letter <= ` ` { return error('"${value}" is not a Kepr address — try kepr.uk') } } text := if value.starts_with('http://') || value.starts_with('https://') { value } else { 'https://${value}' } parsed := urllib.parse(text) or { return error('"${value}" is not a Kepr address — try kepr.uk') } if parsed.scheme !in ['http', 'https'] || parsed.host == '' || parsed.path.trim_right('/') != '' || parsed.raw_query != '' || parsed.fragment != '' { return error('"${value}" is not a Kepr address — try kepr.uk') } if text.all_after('//').contains('@') { return error('"${value}" is not a Kepr address — try kepr.uk') } // `/api/connect` answers with the bucket's secret key, so plain http is only for a Kepr on // this machine: a test, or a local run. if parsed.scheme == 'http' && parsed.hostname().to_lower() !in ['127.0.0.1', 'localhost', '::1'] { return error('"${value}" needs https — plain http is only for a Kepr on this machine') } // A hostname is case-insensitive, so the stored form is the lower-case one. return '${parsed.scheme}://${parsed.host.to_lower()}' } pub fn (mut w Web) get(key string) !Object { response := http.fetch( url: w.url(key) method: .get enable_http2: false allow_redirect: false )! if response.status_code == 404 { return Object{} } if response.status_code < 200 || response.status_code >= 300 { return error('web GET ${key} failed: ${response.status_code}') } return Object{ found: true body: response.body.bytes() tag: response.header.get_custom('etag', http.HeaderQueryConfig{}) or { '' } } } // stream is get for a body too big to hold: each piece goes to `sink` as it arrives, and none of it // is kept. False when the key does not exist. pub fn (mut w Web) stream(key string, sink fn (piece []u8) !) !bool { on_body := w.on_body response := http.fetch( url: w.url(key) method: .get enable_http2: false allow_redirect: false // Enough for the status line and headers, which are all that is read from the response. stop_copying_limit: 64 * 1024 on_progress_body: fn [sink, on_body] (_ &http.Request, piece []u8, received u64, _ u64, status int) ! { // V hands over a read that holds only part of the headers as if it were body, with // nothing received yet; a real piece of body always counts itself. if status < 200 || status >= 300 || received == 0 { return } sink(piece)! if on_body != unsafe { nil } && (received - u64(piece.len)) >> 20 != received >> 20 { on_body(received) } } )! if response.status_code == 404 { return false } if response.status_code < 200 || response.status_code >= 300 { return error('web GET ${key} failed: ${response.status_code}') } return true } pub fn (mut w Web) list(prefix string) ![]string { response := http.fetch( url: w.url(prefix) method: .get enable_http2: false allow_redirect: false )! if response.status_code == 404 { return []string{} } if response.status_code < 200 || response.status_code >= 300 { return error('web GET ${prefix} failed: ${response.status_code}') } mut result := []string{} for id in response.body.split_into_lines() { if id.trim_space() != '' { result << '${prefix}/${id.trim_space()}' } } return result } pub fn (mut w Web) put_if_absent(key string, body []u8) !bool { return error('${w.repo(key)} is read-only.') } pub fn (mut w Web) put_if_match(key string, body []u8, tag string) !bool { return error('${w.repo(key)} is read-only.') } pub fn (mut w Web) delete(key string) ! { return error('${w.repo(key)} is read-only.') } fn (w &Web) url(key string) string { parts := key.split('/') return '${w.base}/${parts[0]}/koh/${parts[1..].join('/')}' } fn (w &Web) repo(key string) string { return '${w.base}/${key.all_before('/')}' }