skip and report unsafe paths on save and arrival with one shared rule (3.2.0)

@@ -1 +1 @@1-3.1.51+3.2.0

@@ -26,6 +26,9 @@2626 	defer {2727 		store.close()2828 	}29+	walked := snapshot.walk_report(store, store.root, snapshot.load(store.root)) or {30+		return refuse(args.json(), err.msg())31+	}2932 	save := history.save_worktree(store, store.root, snapshot.load(store.root), history.SaveInput{3033 		message:       message3134 		agent:         args.value('agent')@@ -46,10 +49,14 @@4649 			session:       save.session4750 			files:         files.len4851 			size_bytes:    save.size_bytes52+			unsafe_paths:  walked.unsafe_paths4953 		}))5054 		auto_offer(store, true)5155 		return 05256 	}57+	for path in walked.unsafe_paths {58+		warn('not saved: unsafe name ${path}')59+	}5360 	ok('saved [${save.id}]  ${save.message}')5461 	mut detail := []string{}5562 	if save.is_checkpoint {@@ -86,4 +93,5 @@8693 	session       string8794 	files         int8895 	size_bytes    i6496+	unsafe_paths  []string8997 }

@@ -48,12 +48,16 @@4848 			saves:              line.len4949 			size_bytes:         os.file_size(os.join_path(store.root, storage.file_name))5050 			head_flagged:       head_flag.flagged51+			unsafe_paths:       changes.unsafe_paths5152 		}))5253 		return 05354 	}5455 	if unfinished {5556 		warn(snapshot.restore_unfinished)5657 	}58+	for path in changes.unsafe_paths {59+		warn('not saved: unsafe name ${path}')60+	}5761 	if !changes.dirty {5862 		ok('no changes')5963 		if changes.ignore_changed {@@ -84,6 +88,7 @@8488 	modified       []string8589 	removed        []string8690 	ignore_changed bool91+	unsafe_paths   []string8792 }8893 8994 struct StatusJson {@@ -98,11 +103,13 @@98103 	saves              int99104 	size_bytes         u64100105 	head_flagged       bool106+	unsafe_paths       []string101107 }102108 103109 pub fn compare_with_head(store &storage.Store, head history.Save) !Changes {104110 	before := snapshot.index_files(history.files_of(store, head.id)!)105-	after := snapshot.index_files(snapshot.walk(store, store.root, snapshot.load(store.root))!)111+	walked := snapshot.walk_report(store, store.root, snapshot.load(store.root))!112+	after := snapshot.index_files(walked.files)106113 	mut added := []string{}107114 	mut modified := []string{}108115 	mut removed := []string{}@@ -129,5 +136,6 @@129136 		modified:       modified130137 		removed:        removed131138 		ignore_changed: ignore_changed139+		unsafe_paths:   walked.unsafe_paths132140 	}133141 }

@@ -98,6 +98,7 @@9898 			flagged:       report.flagged.map(it.save_id)9999 			previous_head: report.previous_head100100 			kept_versions: report.kept_versions101+			unsafe_paths:  report.unsafe_paths101102 		}))102103 		return 0103104 	}@@ -137,6 +138,7 @@137138 		note_kept_versions(report)138139 	}139140 	warn_flagged(report.flagged)141+	warn_unsafe_paths(report.unsafe_paths)140142 	if report.clashes.len > 0 {141143 		note('${report.clashes.len} changed on both machines: ${report.clashes.join(', ')}')142144 	}@@ -459,13 +461,14 @@459461 	finished = true460462 	if args.json() {461463 		emit_json(json2.encode(StolenJson{462-			repo:    name463-			forward: true464-			saves:   fetched.saves.len465-			objects: fetched.repo_objects.len466-			events:  fetched.events_out.len467-			head:    fetched.head468-			flagged: flagged.map(it.save_id)464+			repo:         name465+			forward:      true466+			saves:        fetched.saves.len467+			objects:      fetched.repo_objects.len468+			events:       fetched.events_out.len469+			head:         fetched.head470+			flagged:      flagged.map(it.save_id)471+			unsafe_paths: fetched.unsafe_paths469472 		}))470473 		return 0471474 	}@@ -475,6 +478,7 @@475478 		note('read-only: this copy keeps up with ${source} — sign in with "koh remote ${exchange.source_host(source)}" to make it yours')476479 	}477480 	warn_flagged(flagged)481+	warn_unsafe_paths(fetched.unsafe_paths)478482 	return 0479483 }480484 @@ -512,6 +516,12 @@512516 	}513517 }514518 519+fn warn_unsafe_paths(paths []string) {520+	for path in paths {521+		warn('dropped unsafe path ${path}')522+	}523+}524+515525 struct ReposJson {516526 	repos []string517527 }@@ -530,4 +540,5 @@530540 	flagged       []string531541 	previous_head string532542 	kept_versions []string543+	unsafe_paths  []string533544 }

@@ -237,6 +237,7 @@237237 	events_out   []events.Event238238 	head         string239239 	base         string // the newest save both sides already shared, '' when none240+	unsafe_paths []string240241 }241242 242243 pub fn fetch_back_to_known(store &storage.Store, mut backend remote.Backend, repo string) !Fetched {@@ -254,7 +255,7 @@254255 	}255256 	mut cursor := fetched.head256257 	for cursor != '' && cursor !in on_line {257-		record := if saved := history.by_id(store, cursor) {258+		mut record := if saved := history.by_id(store, cursor) {258259 			save_record(store, saved)!259260 		} else {260261 			record_object := backend.get(save_key(repo, cursor))!@@ -263,6 +264,7 @@263264 			}264265 			decode_save(record_object.body)!265266 		}267+		fetched.unsafe_paths << drop_unsafe_paths(mut record)266268 		for file in record.files {267269 			if !store.has_object(file.hash)! && file.hash !in fetched.repo_objects {268270 				blob := backend.get(object_key(repo, file.hash))!@@ -355,7 +357,8 @@355357 		return356358 	}357359 	files := to_files(record).filter(fn [strips, record] (file snapshot.FileEntry) bool {358-		return !strips.any(reaches(it, record.id) && covers(it.path, file.path))360+		return snapshot.legal_path(file.path)361+			&& !strips.any(reaches(it, record.id) && covers(it.path, file.path))359362 	})360363 	history.import_save(store, to_save(record), files)!361364 }

@@ -11,6 +11,20 @@1111 import src.snapshot1212 import src.storage1313 14+fn test_arriving_record_drops_unsafe_paths_and_reports_them() {15+	mut record := SaveRecord{16+		files: [17+			FileRecord{ path: 'safe.txt' },18+			FileRecord{ path: '../outside.txt' },19+			FileRecord{ path: 'bad\\name.txt' },20+			FileRecord{ path: 'Icon\r' },21+		]22+	}23+	dropped := drop_unsafe_paths(mut record)24+	assert record.files.map(it.path) == ['safe.txt']25+	assert dropped == ['../outside.txt', 'Icon\\x0d', 'bad\\\\name.txt']26+}27+1428 // Machines keep their repo in a folder of the same NAME, because the remote key1529 // is the folder's name; that is what makes them the same repo to the remote.1630 fn machine(name string) !string {

@@ -63,7 +63,8 @@6363 	session       string6464 	restores      string6565 	replayed_from string66-	files         []FileRecord66+pub mut:67+	files []FileRecord6768 }6869 6970 // save_record turns a local save into what travels. The file list is part of@@ -100,6 +101,22 @@100101 101102 pub fn decode_save(body []u8) !SaveRecord {102103 	return json2.decode[SaveRecord](body.bytestr())!104+}105+106+// drop_unsafe_paths applies the shared path contract before an arriving record is used.107+pub fn drop_unsafe_paths(mut record SaveRecord) []string {108+	mut kept := []FileRecord{}109+	mut dropped := []string{}110+	for file in record.files {111+		if snapshot.legal_path(file.path) {112+			kept << file113+		} else {114+			dropped << snapshot.escape_path(file.path)115+		}116+	}117+	record.files = kept118+	dropped.sort()119+	return dropped103120 }104121 105122 pub fn to_save(record SaveRecord) history.Save {

@@ -79,6 +79,7 @@7979 	// brings back the tree from before the steal, less anything stripped.8080 	previous_head string8181 	kept_versions []string82+	unsafe_paths  []string8283 }8384 8485 pub const dirty_tree = 'save or checkpoint first — steal needs a clean tree'@@ -119,6 +120,7 @@119120 		fetched_objects: fetched.repo_objects.len120121 		fetched_events:  fetched.events_out.len121122 		head:            fetched.head123+		unsafe_paths:    fetched.unsafe_paths122124 	}123125 	mut local_only := []history.Save{}124126 	if head := head_before {

@@ -34,6 +34,7 @@3434 3535 # OS noise3636 .DS_Store37+Icon\\r3738 Thumbs.db3839 3940 # Logs@@ -140,6 +141,9 @@140141 // would make the tree dirty the moment a save lands, so `status` could never141142 // report a clean tree and every diff would carry the log's own churn.142143 pub fn hard_ignored(rel string) bool {144+	if rel == 'Icon\r' {145+		return true146+	}143147 	if rel == 'LOG.md' {144148 		return true145149 	}

@@ -129,6 +129,33 @@129129 	assert !plain_executable130130 }131131 132+fn test_legal_path_is_the_shared_save_path_rule() {133+	for path in ['src/main.v', 'café/猫.txt', '.kohignore'] {134+		assert legal_path(path), path135+	}136+	for path in ['', '/etc/passwd', 'a//b', './a', 'a/../b', 'a\\b', 'Icon\r', 'a\u0080b',137+		[u8(0xff)].bytestr()] {138+		assert !legal_path(path), path139+	}140+}141+142+fn test_walk_skips_and_reports_unsafe_names() {143+	root := temp_root('unsafe_names')!144+	defer {145+		os.rmdir_all(root) or {}146+	}147+	store := storage.create(root)!148+	defer {149+		store.close()150+	}151+	put(root, 'safe.txt', 'kept\n')!152+	os.write_file('${root}/bad\\name.txt', 'skipped\n')!153+	os.write_file('${root}/Icon\r', 'skipped\n')!154+	walked := walk_report(store, root, Ignore{})!155+	assert walked.files.map(it.path) == ['safe.txt']156+	assert walked.unsafe_paths == ['Icon\\x0d', 'bad\\\\name.txt']157+}158+132159 // The point of nanosecond timestamps: a file rewritten inside the same second133160 // with the same size must not reuse the cached hash, or a save would record134161 // content that is no longer on disk.

@@ -2,6 +2,7 @@22 33 module snapshot44 5+import encoding.utf856 import os67 import time78 import src.storage@@ -18,18 +19,77 @@1819 	executable bool1920 }2021 22+pub struct WalkReport {23+pub:24+	files        []FileEntry25+	unsafe_paths []string26+}27+28+// legal_path is the one rule for every path Koh saves, receives or restores.29+pub fn legal_path(path string) bool {30+	if path == '' || path.starts_with('/') || path.contains('\\') || !utf8.validate_str(path) {31+		return false32+	}33+	if path.split('/').any(it in ['', '.', '..']) {34+		return false35+	}36+	bytes := path.bytes()37+	for i, byte in bytes {38+		if byte < 0x20 || byte == 0x7f || (byte == 0xc2 && i + 1 < bytes.len39+			&& bytes[i + 1] >= 0x80 && bytes[i + 1] <= 0x9f) {40+			return false41+		}42+	}43+	return true44+}45+46+// escape_path makes unsafe names printable without letting controls affect the terminal.47+pub fn escape_path(path string) string {48+	mut escaped := ''49+	bytes := path.bytes()50+	mut i := 051+	for i < bytes.len {52+		byte := bytes[i]53+		if byte == `\\` {54+			escaped += '\\\\'55+		} else if byte < 0x20 || byte == 0x7f {56+			escaped += '\\x${byte.hex()}'57+		} else if byte == 0xc2 && i + 1 < bytes.len && bytes[i + 1] >= 0x8058+			&& bytes[i + 1] <= 0x9f {59+			escaped += '\\u00${bytes[i + 1].hex()}'60+			i++61+		} else if byte >= 0x80 && !utf8.validate_str(bytes[i..].bytestr()) {62+			escaped += '\\x${byte.hex()}'63+		} else {64+			escaped += byte.ascii_str()65+		}66+		i++67+	}68+	return escaped69+}70+2171 // walk hashes every tracked file under `root` and returns them sorted by path.2272 // Symlinks are skipped, as Koh 2 skipped them, so migration stays lossless.2373 pub fn walk(store &storage.Store, root string, ig Ignore) ![]FileEntry {74+	return walk_report(store, root, ig)!.files75+}76+77+pub fn walk_report(store &storage.Store, root string, ig Ignore) !WalkReport {2478 	cache := store.load_cache()!2579 	mut entries := []FileEntry{}2680 	mut fresh := []storage.CacheRow{}27-	walk_dir(root, '', ig.banning(store.stripped_paths()!), cache, mut entries, mut fresh)!81+	mut unsafe_paths := []string{}82+	walk_dir(root, '', ig.banning(store.stripped_paths()!), cache, mut entries, mut fresh,83+		mut unsafe_paths)!2884 	// Skipped while another command writes, or inside a caller's transaction:2985 	// the next walk hashes those files again, which is all a missing row costs.3086 	store.save_cache_rows(fresh) or {}3187 	entries.sort(a.path < b.path)32-	return entries88+	unsafe_paths.sort()89+	return WalkReport{90+		files:        entries91+		unsafe_paths: unsafe_paths92+	}3393 }3494 3595 // walk_raw hashes the same files without a store: no cache, nothing written.@@ -38,15 +98,20 @@3898 pub fn walk_raw(root string, ig Ignore) ![]FileEntry {3999 	mut entries := []FileEntry{}40100 	mut unused := []storage.CacheRow{}41-	walk_dir(root, '', ig, map[string]storage.CacheRow{}, mut entries, mut unused)!101+	mut unsafe_paths := []string{}102+	walk_dir(root, '', ig, map[string]storage.CacheRow{}, mut entries, mut unused, mut unsafe_paths)!42103 	entries.sort(a.path < b.path)43104 	return entries44105 }45106 46-fn walk_dir(root string, rel string, ig Ignore, cache map[string]storage.CacheRow, mut out []FileEntry, mut fresh []storage.CacheRow) ! {107+fn walk_dir(root string, rel string, ig Ignore, cache map[string]storage.CacheRow, mut out []FileEntry, mut fresh []storage.CacheRow, mut unsafe_paths []string) ! {47108 	dir := if rel == '' { root } else { os.join_path(root, rel) }48109 	for name in os.ls(dir)! {49110 		child := if rel == '' { name } else { '${rel}/${name}' }111+		if !legal_path(child) {112+			unsafe_paths << escape_path(child)113+			continue114+		}50115 		st := lstat(os.join_path(root, child)) or { continue }51116 		// The kind is known by now, so a pattern ending in `/` can be applied52117 		// to directories only.@@ -55,7 +120,7 @@55120 		}56121 		match st.kind {57122 			.dir {58-				walk_dir(root, child, ig, cache, mut out, mut fresh)!123+				walk_dir(root, child, ig, cache, mut out, mut fresh, mut unsafe_paths)!59124 			}60125 			.file {61126 				if st.size > storage.max_file_size {

@@ -384,6 +384,23 @@384384 	assert !run(dir, 'status').out.contains('LOG.md')385385 }386386 387+fn test_status_and_save_report_unsafe_names_without_refusing_the_save() {388+	dir := temp_repo('unsafe-names')389+	defer { os.rmdir_all(dir) or {} }390+	init_repo(dir)391+	write(dir, 'safe.txt', 'safe\n')392+	assert run(dir, 'save', 'safe').code == 0393+	os.write_file('${dir}/bad\\name.txt', 'bad\n') or { panic(err) }394+	os.write_file('${dir}/Icon\r', 'icon\n') or { panic(err) }395+396+	status := run(dir, 'status', '--json')397+	assert status.code == 0398+	assert status.out.contains('"unsafe_paths":["Icon\\\\x0d","bad\\\\\\\\name.txt"]'), status.out399+	saved := run(dir, 'checkpoint', 'unsafe names', '--json')400+	assert saved.code == 0401+	assert saved.out.contains('"unsafe_paths":["Icon\\\\x0d","bad\\\\\\\\name.txt"]'), saved.out402+}403+387404 // LOG.md is a convenience written after the save lands. If it cannot be388405 // written, the save still happened, and an agent that retried on a failure389406 // would only be told nothing had changed.

@@ -1,6 +1,6 @@11 Module {22 	name: 'koh3'3-	version: '3.1.5'3+	version: '3.2.0'44 	description: 'One SQLite file per repo, one line of history.'55 	license: 'MIT'66 }