FILEOPS invariant oracle for v10 ================================ Invariant checked: after any mix of commit, rollback, crash, checkpoint, 2PC and standby replay, the files FILEOPS manages are exactly those the committed transactions imply. Scenarios: the full product of operation create, mkdir, link, rename, write, delete (via test_fileops) end commit, rollback, rollback to savepoint, crash with the transaction in flight, crash after the abort record is flushed (injection point fileops-abort-after-flush), commit prepared, rollback prepared checkpoint none, before the operation, between operation and end reuse another transaction takes the freed name afterwards: no/yes minus the combinations where no name is free to reuse: 186 scenarios. Every scenario is checked on the primary right after it ends (non-crash ends), and on the primary and the standby after one crash and again after a second crash of the primary. Usage: python3 oracle.py gen scenarios.json SCENARIOS=scenarios.json DUMP_DIR=out prove 910_oracle_runner.pl python3 oracle.py check scenarios.json out1 out2 out3 The model in oracle.py knows nothing about FILEOPS internals; the runner only executes and dumps. At most 6 aborts are paused per crash round because the injection_points module has 8 wait slots. Results on v10 (three runs, same 21 failures each time, none unstable) --------------------------------------------------------------------- scenarios 186 runs 3 pass 165 fail 21 unstable 0 All 21 failures are crashes with the transaction in flight, seen on the standby; the primary matches the model in all 186 scenarios. FAIL s015 op=create end=crash_in_flight ckpt=none reuse=no after_crash/standby/s015_f: want -:None got f: after_second_crash/standby/s015_f: want -:None got f: FAIL s017 op=create end=crash_in_flight ckpt=before reuse=no after_crash/standby/s017_f: want -:None got f: after_second_crash/standby/s017_f: want -:None got f: FAIL s019 op=create end=crash_in_flight ckpt=between reuse=no after_crash/standby/s019_f: want -:None got f: after_second_crash/standby/s019_f: want -:None got f: FAIL s051 op=mkdir end=crash_in_flight ckpt=none reuse=no after_crash/standby/s051_d: want -:None got d:None after_second_crash/standby/s051_d: want -:None got d:None FAIL s053 op=mkdir end=crash_in_flight ckpt=before reuse=no after_crash/standby/s053_d: want -:None got d:None after_second_crash/standby/s053_d: want -:None got d:None FAIL s055 op=mkdir end=crash_in_flight ckpt=between reuse=no after_crash/standby/s055_d: want -:None got d:None after_second_crash/standby/s055_d: want -:None got d:None FAIL s087 op=link end=crash_in_flight ckpt=none reuse=no after_crash/standby/s087_l: want -:None got f:committed-original after_second_crash/standby/s087_l: want -:None got f:committed-original FAIL s088 op=link end=crash_in_flight ckpt=none reuse=yes after_crash/standby/s088_src: want f:committed-original got f:reused-by-another-xact after_second_crash/standby/s088_src: want f:committed-original got f:reused-by-another-xact FAIL s089 op=link end=crash_in_flight ckpt=before reuse=no after_crash/standby/s089_l: want -:None got f:committed-original after_second_crash/standby/s089_l: want -:None got f:committed-original FAIL s090 op=link end=crash_in_flight ckpt=before reuse=yes after_crash/standby/s090_src: want f:committed-original got f:reused-by-another-xact after_second_crash/standby/s090_src: want f:committed-original got f:reused-by-another-xact FAIL s091 op=link end=crash_in_flight ckpt=between reuse=no after_crash/standby/s091_l: want -:None got f:committed-original after_second_crash/standby/s091_l: want -:None got f:committed-original FAIL s092 op=link end=crash_in_flight ckpt=between reuse=yes after_crash/standby/s092_src: want f:committed-original got f:reused-by-another-xact after_second_crash/standby/s092_src: want f:committed-original got f:reused-by-another-xact FAIL s123 op=rename end=crash_in_flight ckpt=none reuse=no after_crash/standby/s123_a: want f:committed-original got -:None after_crash/standby/s123_b: want -:None got f:committed-original after_second_crash/standby/s123_a: want f:committed-original got -:None after_second_crash/standby/s123_b: want -:None got f:committed-original FAIL s124 op=rename end=crash_in_flight ckpt=none reuse=yes after_crash/standby/s124_a: want f:committed-original got -:None after_second_crash/standby/s124_a: want f:committed-original got -:None FAIL s125 op=rename end=crash_in_flight ckpt=before reuse=no after_crash/standby/s125_a: want f:committed-original got -:None after_crash/standby/s125_b: want -:None got f:committed-original after_second_crash/standby/s125_a: want f:committed-original got -:None after_second_crash/standby/s125_b: want -:None got f:committed-original FAIL s126 op=rename end=crash_in_flight ckpt=before reuse=yes after_crash/standby/s126_a: want f:committed-original got -:None after_second_crash/standby/s126_a: want f:committed-original got -:None FAIL s127 op=rename end=crash_in_flight ckpt=between reuse=no after_crash/standby/s127_a: want f:committed-original got -:None after_crash/standby/s127_b: want -:None got f:committed-original after_second_crash/standby/s127_a: want f:committed-original got -:None after_second_crash/standby/s127_b: want -:None got f:committed-original FAIL s128 op=rename end=crash_in_flight ckpt=between reuse=yes after_crash/standby/s128_a: want f:committed-original got -:None after_second_crash/standby/s128_a: want f:committed-original got -:None FAIL s153 op=write end=crash_in_flight ckpt=none reuse=no after_crash/standby/s153_w: want f:committed-original got f:NEW-BYTES-original after_second_crash/standby/s153_w: want f:committed-original got f:NEW-BYTES-original FAIL s154 op=write end=crash_in_flight ckpt=before reuse=no after_crash/standby/s154_w: want f:committed-original got f:NEW-BYTES-original after_second_crash/standby/s154_w: want f:committed-original got f:NEW-BYTES-original FAIL s155 op=write end=crash_in_flight ckpt=between reuse=no after_crash/standby/s155_w: want f:committed-original got f:NEW-BYTES-original after_second_crash/standby/s155_w: want f:committed-original got f:NEW-BYTES-original ===== oracle.py ===== """Scenario generator, model and oracle for the FILEOPS invariant. Invariant: after any mix of commit, rollback, crash, checkpoint, 2PC and standby replay, the files FILEOPS manages are exactly those the committed transactions imply. python3 -I oracle.py gen python3 -I oracle.py check [ ...] The model below knows nothing about FILEOPS internals: it computes the expected state of each scenario's names from the history alone. The runner (t/910_oracle_runner.pl) executes the scenarios and dumps what is on disk; `check` compares the two at every stage, on the primary and on the standby. """ import itertools import json import sys from collections import defaultdict OPS = ["create", "mkdir", "link", "rename", "write", "delete"] ENDS = ["commit", "rollback", "rollback_to_savepoint", "crash_in_flight", "crash_after_abort_flush", "commit_prepared", "rollback_prepared"] CKPTS = ["none", "before", "between"] REUSE = ["no", "yes"] CRASH_ENDS = {"crash_in_flight", "crash_after_abort_flush"} COMMITTED_ENDS = {"commit", "commit_prepared"} SETUP_CONTENT = "committed-original" WRITE_CONTENT = "NEW-BYTES" REUSE_CONTENT = "reused-by-another-xact" def reuse_target(op, committed): """The name that is free after the end, for another transaction to take.""" if committed: return None return {"create": "f", "mkdir": "d", "link": "l", "rename": "b"}.get(op) def scenarios(): out = [] for i, (op, end, ckpt, reuse) in enumerate(itertools.product(OPS, ENDS, CKPTS, REUSE)): committed = end in COMMITTED_ENDS if reuse == "yes" and reuse_target(op, committed) is None: continue # nothing is free to reuse out.append({"id": f"s{len(out):03d}", "op": op, "end": end, "ckpt": ckpt, "reuse": reuse, "reuse_name": reuse_target(op, committed)}) return out def model(s): """Expected state of the scenario's names: {name: (type, content)}. type is 'f' (file), 'd' (directory) or '-' (absent).""" committed = s["end"] in COMMITTED_ENDS n = lambda x: f"{s['id']}_{x}" st = {} op = s["op"] if op == "create": st[n("f")] = ("f", "") if committed else ("-", None) elif op == "mkdir": st[n("d")] = ("d", None) if committed else ("-", None) elif op == "link": st[n("src")] = ("f", SETUP_CONTENT) st[n("l")] = ("f", SETUP_CONTENT) if committed else ("-", None) elif op == "rename": st[n("a")] = ("-", None) if committed else ("f", SETUP_CONTENT) st[n("b")] = ("f", SETUP_CONTENT) if committed else ("-", None) elif op == "write": st[n("w")] = ("f", WRITE_CONTENT + SETUP_CONTENT[len(WRITE_CONTENT):] if committed else SETUP_CONTENT) elif op == "delete": st[n("x")] = ("-", None) if committed else ("f", SETUP_CONTENT) if s["reuse"] == "yes": name = n(s["reuse_name"]) st[name] = ("d", None) if op == "mkdir" else ("f", REUSE_CONTENT) return st def stages_for(s): """Stages at which the scenario's state is defined. 'online' is the primary right after the end; crash scenarios only exist after the crash.""" first = [] if s["end"] in CRASH_ENDS else [("online", "primary")] return first + [(st, node) for st in ("after_crash", "after_second_crash") for node in ("primary", "standby")] def read_dump(path): """Lines: """ d = defaultdict(dict) for line in open(path): stage, node, name, typ, content = line.rstrip("\n").split("\t") d[(stage, node)][name] = (typ, None if content == "\\N" else content) return d def check(spec, dumps): scen = json.load(open(spec)) runs = [read_dump(f"{d}/dump.tsv") for d in dumps] result = {} for s in scen: want = model(s) verdicts = [] for run in runs: bad = [] for stage, node in stages_for(s): got = run.get((stage, node), {}) for name, w in want.items(): g = got.get(name, ("-", None)) if (g[0], g[1] if w[0] == "f" else None) != (w[0], w[1] if w[0] == "f" else None): bad.append(f"{stage}/{node}/{name}: want {w[0]}:{w[1]} got {g[0]}:{g[1]}") verdicts.append(bad) result[s["id"]] = (s, verdicts) stable_fail, flaky, ok = [], [], [] for sid, (s, v) in result.items(): fails = [bool(b) for b in v] if all(fails): stable_fail.append((s, v[0])) elif any(fails): flaky.append((s, [b for b in v if b][0])) else: ok.append(s) print(f"scenarios {len(result)} runs {len(runs)} pass {len(ok)} " f"fail {len(stable_fail)} unstable {len(flaky)}") for label, group in (("FAIL", stable_fail), ("UNSTABLE", flaky)): for s, bad in group: print(f"{label} {s['id']} op={s['op']} end={s['end']} ckpt={s['ckpt']} reuse={s['reuse']}") for b in bad[:4]: print(f" {b}") if __name__ == "__main__": if sys.argv[1] == "gen": sc = scenarios() json.dump(sc, open(sys.argv[2], "w"), indent=1) print(f"{len(sc)} scenarios") elif sys.argv[1] == "check": check(sys.argv[2], sys.argv[3:]) ===== 910_oracle_runner.pl ===== # Runner for the FILEOPS invariant oracle (oracle.py). Executes every # scenario in SCENARIOS (a JSON file from `oracle.py gen`) on a primary with # a streaming standby, and writes what is on disk for each scenario's names to # $ENV{DUMP_DIR}/dump.tsv at three stages: # online the primary right after each non-crash scenario ends # after_crash primary and standby after one crash of the primary # after_second_crash both again after a second crash (replays the same WAL) # The runner does not judge anything; oracle.py compares against its model. use strict; use warnings FATAL => 'all'; use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; use JSON::PP; use Time::HiRes qw(usleep); my $scen = decode_json(slurp_file($ENV{SCENARIOS})); my $dumpdir = $ENV{DUMP_DIR} or die 'DUMP_DIR'; open(my $dump, '>', "$dumpdir/dump.tsv") or die; my $SETUP = 'committed-original'; my $NEW = 'NEW-BYTES'; my $REUSE = 'reused-by-another-xact'; my $primary = PostgreSQL::Test::Cluster->new('primary'); $primary->init(allows_streaming => 1); $primary->append_conf('postgresql.conf', qq{ max_connections = 250 max_prepared_transactions = 100 checkpoint_timeout = 1h autovacuum = off }); $primary->start; $primary->safe_psql('postgres', 'CREATE EXTENSION test_fileops; CREATE EXTENSION injection_points'); $primary->backup('b'); my $standby = PostgreSQL::Test::Cluster->new('standby'); $standby->init_from_backup($primary, 'b', has_streaming => 1); $standby->start; sub sql { $primary->safe_psql('postgres', $_[0]) } sub nm { my ($s, $x) = @_; return "$s->{id}_$x" } sub mkfile { my ($name, $content) = @_; sql("BEGIN; SELECT test_fileops_create('$name', 384);" . " SELECT test_fileops_write('$name', 0, '$content'); COMMIT;"); } sub setup { my ($s) = @_; my %fixture = (link => 'src', rename => 'a', write => 'w', delete => 'x'); mkfile(nm($s, $fixture{ $s->{op} }), $SETUP) if $fixture{ $s->{op} }; } sub op_sql { my ($s) = @_; my $o = $s->{op}; return "SELECT test_fileops_create('" . nm($s, 'f') . "', 384)" if $o eq 'create'; return "SELECT test_fileops_mkdir('" . nm($s, 'd') . "', 448)" if $o eq 'mkdir'; return "SELECT test_fileops_link('" . nm($s, 'src') . "', '" . nm($s, 'l') . "')" if $o eq 'link'; return "SELECT test_fileops_rename('" . nm($s, 'a') . "', '" . nm($s, 'b') . "')" if $o eq 'rename'; return "SELECT test_fileops_write('" . nm($s, 'w') . "', 0, '$NEW')" if $o eq 'write'; return "SELECT test_fileops_delete('" . nm($s, 'x') . "')" if $o eq 'delete'; die $o; } sub reuse { my ($s) = @_; return unless $s->{reuse} eq 'yes'; my $name = nm($s, $s->{reuse_name}); if ($s->{op} eq 'mkdir') { sql("SELECT test_fileops_mkdir('$name', 448)") } else { mkfile($name, $REUSE) } } sub start_xact { my ($s) = @_; sql('CHECKPOINT') if $s->{ckpt} eq 'before'; my $bg = $primary->background_psql('postgres', on_error_stop => 0); $bg->query_safe('BEGIN'); $bg->query_safe('SAVEPOINT sp') if $s->{end} eq 'rollback_to_savepoint'; $bg->query_safe(op_sql($s)); sql('CHECKPOINT') if $s->{ckpt} eq 'between'; return $bg; } sub dump_state { my ($stage, $node) = @_; my $dir = $node->data_dir; opendir(my $d, $dir) or die; for my $e (sort grep { /^s\d{3}_/ } readdir($d)) { my $p = "$dir/$e"; my ($t, $c) = -d $p ? ('d', '\N') : ('f', slurp_file($p)); print $dump join("\t", $stage, $node->name, $e, $t, $c), "\n"; } closedir($d); } my @crash = grep { $_->{end} =~ /^crash_/ } @$scen; my @plain = grep { $_->{end} !~ /^crash_/ } @$scen; setup($_) for @$scen; # Non-crash scenarios, one at a time. for my $s (@plain) { my $bg = start_xact($s); my $e = $s->{end}; if ($e eq 'commit') { $bg->query_safe('COMMIT') } elsif ($e eq 'rollback') { $bg->query_safe('ROLLBACK') } elsif ($e eq 'rollback_to_savepoint') { $bg->query_safe('ROLLBACK TO sp; COMMIT') } else { $bg->query_safe("PREPARE TRANSACTION 'g$s->{id}'"); sql(($e eq 'commit_prepared' ? 'COMMIT' : 'ROLLBACK') . " PREPARED 'g$s->{id}'"); } $bg->quit; reuse($s); } dump_state('online', $primary); # Crash scenarios, in rounds that each end with one crash. The injection # point module has 8 wait slots, so at most 6 aborts are paused per round; # the in-flight losers all go in the first round. my @after = grep { $_->{end} eq 'crash_after_abort_flush' } @crash; my @inflight = grep { $_->{end} eq 'crash_in_flight' } @crash; my @rounds = ([ @inflight, splice(@after, 0, 6) ]); push @rounds, [ splice(@after, 0, 6) ] while @after; for my $round (@rounds) { my @open = map { [ $_, start_xact($_) ] } @$round; sql("SELECT injection_points_attach('fileops-abort-after-flush', 'wait')"); my $waiting = 0; for my $o (@open) { next unless $o->[0]{end} eq 'crash_after_abort_flush'; $o->[1]->query_until(qr/go/, "\\echo go\nROLLBACK;\n"); # A deferred delete has no inverse, so its abort does not pause. $waiting++ unless $o->[0]{op} eq 'delete'; } $primary->poll_query_until('postgres', "SELECT count(*) = $waiting FROM pg_stat_activity WHERE wait_event = 'fileops-abort-after-flush'") or die 'aborts did not pause'; note "crash round: " . scalar(@open) . " scenarios, paused after abort flush: $waiting"; $primary->stop('immediate'); eval { $_->[1]->quit } for @open; $primary->start; reuse($_) for @$round; } $primary->wait_for_catchup($standby); dump_state('after_crash', $_) for ($primary, $standby); $primary->stop('immediate'); $primary->start; sql('SELECT 1'); $primary->wait_for_catchup($standby); dump_state('after_second_crash', $_) for ($primary, $standby); close($dump); ok(1, 'runner finished'); $standby->stop; $primary->stop; done_testing();