diff --git a/cts/scheduler/xml/bug-1572-1.xml b/cts/scheduler/xml/bug-1572-1.xml
index ab8697e3d4..5d03bd1417 100644
--- a/cts/scheduler/xml/bug-1572-1.xml
+++ b/cts/scheduler/xml/bug-1572-1.xml
@@ -1,192 +1,189 @@
-
-
-
-
+
diff --git a/cts/scheduler/xml/bug-1572-2.xml b/cts/scheduler/xml/bug-1572-2.xml
index a445f12a72..e2c92598e3 100644
--- a/cts/scheduler/xml/bug-1572-2.xml
+++ b/cts/scheduler/xml/bug-1572-2.xml
@@ -1,181 +1,178 @@
-
-
-
-
+
diff --git a/cts/scheduler/xml/inc6.xml b/cts/scheduler/xml/inc6.xml
index 10c06550d9..0dca8ffe67 100644
--- a/cts/scheduler/xml/inc6.xml
+++ b/cts/scheduler/xml/inc6.xml
@@ -1,275 +1,271 @@
-
-
-
-
-
+
-
+
diff --git a/cts/scheduler/xml/order6.xml b/cts/scheduler/xml/order6.xml
index 3e207b8334..c648c2ba5c 100644
--- a/cts/scheduler/xml/order6.xml
+++ b/cts/scheduler/xml/order6.xml
@@ -1,86 +1,70 @@
-
-
-
-
-
-
-
-
-
-
+
+
-
-
-
-
-
-
-
-
-
-
+
+
-
+
-
+
diff --git a/cts/scheduler/xml/rec-node-11.xml b/cts/scheduler/xml/rec-node-11.xml
index 0743619182..8d19aecc92 100644
--- a/cts/scheduler/xml/rec-node-11.xml
+++ b/cts/scheduler/xml/rec-node-11.xml
@@ -1,63 +1,56 @@
-
-
-
-
-
-
-
-
+
-
+
diff --git a/cts/scheduler/xml/rec-rsc-9.xml b/cts/scheduler/xml/rec-rsc-9.xml
index 866dfd46f7..7099ecf4cd 100644
--- a/cts/scheduler/xml/rec-rsc-9.xml
+++ b/cts/scheduler/xml/rec-rsc-9.xml
@@ -1,63 +1,53 @@
-
-
-
-
-
+
-
-
-
-
-
-
-
-
+
+
diff --git a/cts/schemas/test-3/ref.err/no-validate-with.ref.err-0 b/cts/schemas/test-3/ref.err/can-fail.ref.err-0
similarity index 100%
copy from cts/schemas/test-3/ref.err/no-validate-with.ref.err-0
copy to cts/schemas/test-3/ref.err/can-fail.ref.err-0
diff --git a/cts/schemas/test-3/ref.err/no-validate-with.ref.err-1 b/cts/schemas/test-3/ref.err/can-fail.ref.err-1
similarity index 100%
rename from cts/schemas/test-3/ref.err/no-validate-with.ref.err-1
rename to cts/schemas/test-3/ref.err/can-fail.ref.err-1
diff --git a/cts/schemas/test-3/ref.err/no-validate-with.ref.err-2 b/cts/schemas/test-3/ref.err/can-fail.ref.err-2
similarity index 100%
rename from cts/schemas/test-3/ref.err/no-validate-with.ref.err-2
rename to cts/schemas/test-3/ref.err/can-fail.ref.err-2
diff --git a/cts/schemas/test-3/ref.err/no-validate-with.ref.err-3 b/cts/schemas/test-3/ref.err/can-fail.ref.err-3
similarity index 100%
rename from cts/schemas/test-3/ref.err/no-validate-with.ref.err-3
rename to cts/schemas/test-3/ref.err/can-fail.ref.err-3
diff --git a/cts/schemas/test-3/ref.err/no-validate-with.ref.err-4 b/cts/schemas/test-3/ref.err/can-fail.ref.err-4
similarity index 100%
rename from cts/schemas/test-3/ref.err/no-validate-with.ref.err-4
rename to cts/schemas/test-3/ref.err/can-fail.ref.err-4
diff --git a/cts/schemas/test-3/ref.err/no-validate-with.ref.err-99 b/cts/schemas/test-3/ref.err/can-fail.ref.err-99
similarity index 100%
rename from cts/schemas/test-3/ref.err/no-validate-with.ref.err-99
rename to cts/schemas/test-3/ref.err/can-fail.ref.err-99
diff --git a/cts/schemas/test-3/ref.err/no-validate-with.ref.err-0 b/cts/schemas/test-3/ref.err/restart-type.ref.err-0
similarity index 100%
copy from cts/schemas/test-3/ref.err/no-validate-with.ref.err-0
copy to cts/schemas/test-3/ref.err/restart-type.ref.err-0
diff --git a/cts/schemas/test-3/ref.err/no-validate-with.ref.err-0 b/cts/schemas/test-3/ref.err/restart-type.ref.err-1
similarity index 100%
copy from cts/schemas/test-3/ref.err/no-validate-with.ref.err-0
copy to cts/schemas/test-3/ref.err/restart-type.ref.err-1
diff --git a/cts/schemas/test-3/ref.err/no-validate-with.ref.err-0 b/cts/schemas/test-3/ref.err/restart-type.ref.err-2
similarity index 100%
copy from cts/schemas/test-3/ref.err/no-validate-with.ref.err-0
copy to cts/schemas/test-3/ref.err/restart-type.ref.err-2
diff --git a/cts/schemas/test-3/ref.err/no-validate-with.ref.err-0 b/cts/schemas/test-3/ref.err/restart-type.ref.err-3
similarity index 100%
copy from cts/schemas/test-3/ref.err/no-validate-with.ref.err-0
copy to cts/schemas/test-3/ref.err/restart-type.ref.err-3
diff --git a/cts/schemas/test-3/ref.err/no-validate-with.ref.err-0 b/cts/schemas/test-3/ref.err/restart-type.ref.err-4
similarity index 100%
copy from cts/schemas/test-3/ref.err/no-validate-with.ref.err-0
copy to cts/schemas/test-3/ref.err/restart-type.ref.err-4
diff --git a/cts/schemas/test-3/ref.err/no-validate-with.ref.err-0 b/cts/schemas/test-3/ref.err/restart-type.ref.err-99
similarity index 100%
copy from cts/schemas/test-3/ref.err/no-validate-with.ref.err-0
copy to cts/schemas/test-3/ref.err/restart-type.ref.err-99
diff --git a/cts/schemas/test-3/ref.err/no-validate-with.ref.err-0 b/cts/schemas/test-3/ref.err/role-after-failure.ref.err-0
similarity index 100%
copy from cts/schemas/test-3/ref.err/no-validate-with.ref.err-0
copy to cts/schemas/test-3/ref.err/role-after-failure.ref.err-0
diff --git a/cts/schemas/test-3/ref.err/no-validate-with.ref.err-0 b/cts/schemas/test-3/ref.err/role-after-failure.ref.err-1
similarity index 100%
copy from cts/schemas/test-3/ref.err/no-validate-with.ref.err-0
copy to cts/schemas/test-3/ref.err/role-after-failure.ref.err-1
diff --git a/cts/schemas/test-3/ref.err/no-validate-with.ref.err-0 b/cts/schemas/test-3/ref.err/role-after-failure.ref.err-2
similarity index 100%
copy from cts/schemas/test-3/ref.err/no-validate-with.ref.err-0
copy to cts/schemas/test-3/ref.err/role-after-failure.ref.err-2
diff --git a/cts/schemas/test-3/ref.err/no-validate-with.ref.err-0 b/cts/schemas/test-3/ref.err/role-after-failure.ref.err-3
similarity index 100%
copy from cts/schemas/test-3/ref.err/no-validate-with.ref.err-0
copy to cts/schemas/test-3/ref.err/role-after-failure.ref.err-3
diff --git a/cts/schemas/test-3/ref.err/no-validate-with.ref.err-0 b/cts/schemas/test-3/ref.err/role-after-failure.ref.err-4
similarity index 100%
copy from cts/schemas/test-3/ref.err/no-validate-with.ref.err-0
copy to cts/schemas/test-3/ref.err/role-after-failure.ref.err-4
diff --git a/cts/schemas/test-3/ref.err/no-validate-with.ref.err-0 b/cts/schemas/test-3/ref.err/role-after-failure.ref.err-99
similarity index 100%
rename from cts/schemas/test-3/ref.err/no-validate-with.ref.err-0
rename to cts/schemas/test-3/ref.err/role-after-failure.ref.err-99
diff --git a/cts/schemas/test-3/ref/can-fail.ref-0 b/cts/schemas/test-3/ref/can-fail.ref-0
new file mode 100644
index 0000000000..84a3a89f32
--- /dev/null
+++ b/cts/schemas/test-3/ref/can-fail.ref-0
@@ -0,0 +1,119 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
diff --git a/cts/schemas/test-3/ref/can-fail.ref-1 b/cts/schemas/test-3/ref/can-fail.ref-1
new file mode 100644
index 0000000000..44c1a256ff
--- /dev/null
+++ b/cts/schemas/test-3/ref/can-fail.ref-1
@@ -0,0 +1,119 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
diff --git a/cts/schemas/test-3/ref/can-fail.ref-2 b/cts/schemas/test-3/ref/can-fail.ref-2
new file mode 100644
index 0000000000..44c1a256ff
--- /dev/null
+++ b/cts/schemas/test-3/ref/can-fail.ref-2
@@ -0,0 +1,119 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
diff --git a/cts/schemas/test-3/ref/can-fail.ref-3 b/cts/schemas/test-3/ref/can-fail.ref-3
new file mode 100644
index 0000000000..44c1a256ff
--- /dev/null
+++ b/cts/schemas/test-3/ref/can-fail.ref-3
@@ -0,0 +1,119 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
diff --git a/cts/schemas/test-3/ref/can-fail.ref-4 b/cts/schemas/test-3/ref/can-fail.ref-4
new file mode 100644
index 0000000000..8bcb6b6fd0
--- /dev/null
+++ b/cts/schemas/test-3/ref/can-fail.ref-4
@@ -0,0 +1,112 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
diff --git a/cts/schemas/test-3/ref/can-fail.ref-99 b/cts/schemas/test-3/ref/can-fail.ref-99
new file mode 100644
index 0000000000..de66b82e09
--- /dev/null
+++ b/cts/schemas/test-3/ref/can-fail.ref-99
@@ -0,0 +1,112 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
diff --git a/cts/schemas/test-3/ref/no-validate-with.ref-0 b/cts/schemas/test-3/ref/no-validate-with.ref-0
deleted file mode 100644
index 37a26ffb89..0000000000
--- a/cts/schemas/test-3/ref/no-validate-with.ref-0
+++ /dev/null
@@ -1,15 +0,0 @@
-
-
-
-
-
-
-
-
-
-
diff --git a/cts/schemas/test-3/ref/no-validate-with.ref-1 b/cts/schemas/test-3/ref/no-validate-with.ref-1
deleted file mode 100644
index 392eb408f3..0000000000
--- a/cts/schemas/test-3/ref/no-validate-with.ref-1
+++ /dev/null
@@ -1,15 +0,0 @@
-
-
-
-
-
-
-
-
-
-
diff --git a/cts/schemas/test-3/ref/no-validate-with.ref-2 b/cts/schemas/test-3/ref/no-validate-with.ref-2
deleted file mode 100644
index 392eb408f3..0000000000
--- a/cts/schemas/test-3/ref/no-validate-with.ref-2
+++ /dev/null
@@ -1,15 +0,0 @@
-
-
-
-
-
-
-
-
-
-
diff --git a/cts/schemas/test-3/ref/no-validate-with.ref-3 b/cts/schemas/test-3/ref/no-validate-with.ref-3
deleted file mode 100644
index 392eb408f3..0000000000
--- a/cts/schemas/test-3/ref/no-validate-with.ref-3
+++ /dev/null
@@ -1,15 +0,0 @@
-
-
-
-
-
-
-
-
-
-
diff --git a/cts/schemas/test-3/ref/no-validate-with.ref-4 b/cts/schemas/test-3/ref/no-validate-with.ref-4
deleted file mode 100644
index 392eb408f3..0000000000
--- a/cts/schemas/test-3/ref/no-validate-with.ref-4
+++ /dev/null
@@ -1,15 +0,0 @@
-
-
-
-
-
-
-
-
-
-
diff --git a/cts/schemas/test-3/ref/no-validate-with.ref-99 b/cts/schemas/test-3/ref/no-validate-with.ref-99
deleted file mode 100644
index 89ed5f8b11..0000000000
--- a/cts/schemas/test-3/ref/no-validate-with.ref-99
+++ /dev/null
@@ -1,15 +0,0 @@
-
-
-
-
-
-
-
-
-
-
diff --git a/cts/schemas/test-3/ref/restart-type.ref-0 b/cts/schemas/test-3/ref/restart-type.ref-0
new file mode 100644
index 0000000000..be80b80728
--- /dev/null
+++ b/cts/schemas/test-3/ref/restart-type.ref-0
@@ -0,0 +1,98 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
diff --git a/cts/schemas/test-3/ref/restart-type.ref-1 b/cts/schemas/test-3/ref/restart-type.ref-1
new file mode 100644
index 0000000000..514867ff8a
--- /dev/null
+++ b/cts/schemas/test-3/ref/restart-type.ref-1
@@ -0,0 +1,98 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
diff --git a/cts/schemas/test-3/ref/restart-type.ref-2 b/cts/schemas/test-3/ref/restart-type.ref-2
new file mode 100644
index 0000000000..514867ff8a
--- /dev/null
+++ b/cts/schemas/test-3/ref/restart-type.ref-2
@@ -0,0 +1,98 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
diff --git a/cts/schemas/test-3/ref/restart-type.ref-3 b/cts/schemas/test-3/ref/restart-type.ref-3
new file mode 100644
index 0000000000..514867ff8a
--- /dev/null
+++ b/cts/schemas/test-3/ref/restart-type.ref-3
@@ -0,0 +1,98 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
diff --git a/cts/schemas/test-3/ref/restart-type.ref-4 b/cts/schemas/test-3/ref/restart-type.ref-4
new file mode 100644
index 0000000000..a126b04395
--- /dev/null
+++ b/cts/schemas/test-3/ref/restart-type.ref-4
@@ -0,0 +1,86 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
diff --git a/cts/schemas/test-3/ref/restart-type.ref-99 b/cts/schemas/test-3/ref/restart-type.ref-99
new file mode 100644
index 0000000000..fb2d3f443e
--- /dev/null
+++ b/cts/schemas/test-3/ref/restart-type.ref-99
@@ -0,0 +1,86 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
diff --git a/cts/schemas/test-3/ref/role-after-failure.ref-0 b/cts/schemas/test-3/ref/role-after-failure.ref-0
new file mode 100644
index 0000000000..4fbab40e25
--- /dev/null
+++ b/cts/schemas/test-3/ref/role-after-failure.ref-0
@@ -0,0 +1,120 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
diff --git a/cts/schemas/test-3/ref/role-after-failure.ref-1 b/cts/schemas/test-3/ref/role-after-failure.ref-1
new file mode 100644
index 0000000000..b79b10f6f9
--- /dev/null
+++ b/cts/schemas/test-3/ref/role-after-failure.ref-1
@@ -0,0 +1,120 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
diff --git a/cts/schemas/test-3/ref/role-after-failure.ref-2 b/cts/schemas/test-3/ref/role-after-failure.ref-2
new file mode 100644
index 0000000000..b79b10f6f9
--- /dev/null
+++ b/cts/schemas/test-3/ref/role-after-failure.ref-2
@@ -0,0 +1,120 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
diff --git a/cts/schemas/test-3/ref/role-after-failure.ref-3 b/cts/schemas/test-3/ref/role-after-failure.ref-3
new file mode 100644
index 0000000000..b79b10f6f9
--- /dev/null
+++ b/cts/schemas/test-3/ref/role-after-failure.ref-3
@@ -0,0 +1,120 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
diff --git a/cts/schemas/test-3/ref/role-after-failure.ref-4 b/cts/schemas/test-3/ref/role-after-failure.ref-4
new file mode 100644
index 0000000000..20d2bd7c57
--- /dev/null
+++ b/cts/schemas/test-3/ref/role-after-failure.ref-4
@@ -0,0 +1,113 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
diff --git a/cts/schemas/test-3/ref/role-after-failure.ref-99 b/cts/schemas/test-3/ref/role-after-failure.ref-99
new file mode 100644
index 0000000000..e9037826d8
--- /dev/null
+++ b/cts/schemas/test-3/ref/role-after-failure.ref-99
@@ -0,0 +1,113 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
diff --git a/cts/schemas/test-3/xml/can-fail.xml b/cts/schemas/test-3/xml/can-fail.xml
new file mode 100644
index 0000000000..42e58d0637
--- /dev/null
+++ b/cts/schemas/test-3/xml/can-fail.xml
@@ -0,0 +1,119 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
diff --git a/cts/schemas/test-3/xml/no-validate-with.xml b/cts/schemas/test-3/xml/no-validate-with.xml
deleted file mode 100644
index ba238aaf92..0000000000
--- a/cts/schemas/test-3/xml/no-validate-with.xml
+++ /dev/null
@@ -1,15 +0,0 @@
-
-
-
-
-
-
-
-
-
-
diff --git a/cts/schemas/test-3/xml/restart-type.xml b/cts/schemas/test-3/xml/restart-type.xml
new file mode 100644
index 0000000000..2b66d3b449
--- /dev/null
+++ b/cts/schemas/test-3/xml/restart-type.xml
@@ -0,0 +1,98 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
diff --git a/cts/schemas/test-3/xml/role-after-failure.xml b/cts/schemas/test-3/xml/role-after-failure.xml
new file mode 100644
index 0000000000..12a93b7130
--- /dev/null
+++ b/cts/schemas/test-3/xml/role-after-failure.xml
@@ -0,0 +1,120 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
diff --git a/lib/pacemaker/pcmk_graph_consumer.c b/lib/pacemaker/pcmk_graph_consumer.c
index c663c0a294..851d73b829 100644
--- a/lib/pacemaker/pcmk_graph_consumer.c
+++ b/lib/pacemaker/pcmk_graph_consumer.c
@@ -1,883 +1,884 @@
/*
* Copyright 2004-2024 the Pacemaker project contributors
*
* The version control history for this file may have further details.
*
* This source code is licensed under the GNU Lesser General Public License
* version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
*/
#include
#include
#include
#include
#include
#include
#include
#include
/*
* Functions for freeing transition graph objects
*/
/*!
* \internal
* \brief Free a transition graph action object
*
* \param[in,out] user_data Action to free
*/
static void
free_graph_action(gpointer user_data)
{
pcmk__graph_action_t *action = user_data;
if (action->timer != 0) {
crm_warn("Cancelling timer for graph action %d", action->id);
g_source_remove(action->timer);
}
if (action->params != NULL) {
g_hash_table_destroy(action->params);
}
pcmk__xml_free(action->xml);
free(action);
}
/*!
* \internal
* \brief Free a transition graph synapse object
*
* \param[in,out] user_data Synapse to free
*/
static void
free_graph_synapse(gpointer user_data)
{
pcmk__graph_synapse_t *synapse = user_data;
g_list_free_full(synapse->actions, free_graph_action);
g_list_free_full(synapse->inputs, free_graph_action);
free(synapse);
}
/*!
* \internal
* \brief Free a transition graph object
*
* \param[in,out] graph Transition graph to free
*/
void
pcmk__free_graph(pcmk__graph_t *graph)
{
if (graph != NULL) {
g_list_free_full(graph->synapses, free_graph_synapse);
free(graph->source);
free(graph->failed_stop_offset);
free(graph->failed_start_offset);
free(graph);
}
}
/*
* Functions for updating graph
*/
/*!
* \internal
* \brief Update synapse after completed prerequisite
*
* A synapse is ready to be executed once all its prerequisite actions (inputs)
* complete. Given a completed action, check whether it is an input for a given
* synapse, and if so, mark the input as confirmed, and mark the synapse as
* ready if appropriate.
*
* \param[in,out] synapse Transition graph synapse to update
* \param[in] action_id ID of an action that completed
*
* \note The only substantial effect here is confirming synapse inputs.
* should_fire_synapse() will recalculate pcmk__synapse_ready, so the only
* thing that uses the pcmk__synapse_ready from here is
* synapse_state_str().
*/
static void
update_synapse_ready(pcmk__graph_synapse_t *synapse, int action_id)
{
if (pcmk_is_set(synapse->flags, pcmk__synapse_ready)) {
return; // All inputs have already been confirmed
}
// Presume ready until proven otherwise
pcmk__set_synapse_flags(synapse, pcmk__synapse_ready);
for (GList *lpc = synapse->inputs; lpc != NULL; lpc = lpc->next) {
pcmk__graph_action_t *prereq = (pcmk__graph_action_t *) lpc->data;
if (prereq->id == action_id) {
crm_trace("Confirming input %d of synapse %d",
action_id, synapse->id);
pcmk__set_graph_action_flags(prereq, pcmk__graph_action_confirmed);
} else if (!pcmk_is_set(prereq->flags, pcmk__graph_action_confirmed)) {
pcmk__clear_synapse_flags(synapse, pcmk__synapse_ready);
crm_trace("Synapse %d still not ready after action %d",
synapse->id, action_id);
}
}
if (pcmk_is_set(synapse->flags, pcmk__synapse_ready)) {
crm_trace("Synapse %d is now ready to execute", synapse->id);
}
}
/*!
* \internal
* \brief Update action and synapse confirmation after action completion
*
* \param[in,out] synapse Transition graph synapse that action belongs to
* \param[in] action_id ID of action that completed
*/
static void
update_synapse_confirmed(pcmk__graph_synapse_t *synapse, int action_id)
{
bool all_confirmed = true;
for (GList *lpc = synapse->actions; lpc != NULL; lpc = lpc->next) {
pcmk__graph_action_t *action = (pcmk__graph_action_t *) lpc->data;
if (action->id == action_id) {
crm_trace("Confirmed action %d of synapse %d",
action_id, synapse->id);
pcmk__set_graph_action_flags(action, pcmk__graph_action_confirmed);
} else if (all_confirmed &&
!pcmk_is_set(action->flags, pcmk__graph_action_confirmed)) {
all_confirmed = false;
crm_trace("Synapse %d still not confirmed after action %d",
synapse->id, action_id);
}
}
if (all_confirmed
&& !pcmk_is_set(synapse->flags, pcmk__synapse_confirmed)) {
crm_trace("Confirmed synapse %d", synapse->id);
pcmk__set_synapse_flags(synapse, pcmk__synapse_confirmed);
}
}
/*!
* \internal
* \brief Update the transition graph with a completed action result
*
* \param[in,out] graph Transition graph to update
* \param[in] action Action that completed
*/
void
pcmk__update_graph(pcmk__graph_t *graph, const pcmk__graph_action_t *action)
{
for (GList *lpc = graph->synapses; lpc != NULL; lpc = lpc->next) {
pcmk__graph_synapse_t *synapse = (pcmk__graph_synapse_t *) lpc->data;
if (pcmk_any_flags_set(synapse->flags,
pcmk__synapse_confirmed|pcmk__synapse_failed)) {
continue; // This synapse already completed
} else if (pcmk_is_set(synapse->flags, pcmk__synapse_executed)) {
update_synapse_confirmed(synapse, action->id);
} else if (!pcmk_is_set(action->flags, pcmk__graph_action_failed)
|| (synapse->priority == PCMK_SCORE_INFINITY)) {
update_synapse_ready(synapse, action->id);
}
}
}
/*
* Functions for executing graph
*/
/* A transition graph consists of various types of actions. The library caller
* registers execution functions for each action type, which will be stored
* here.
*/
static pcmk__graph_functions_t *graph_fns = NULL;
/*!
* \internal
* \brief Set transition graph execution functions
*
* \param[in] Execution functions to use
*/
void
pcmk__set_graph_functions(pcmk__graph_functions_t *fns)
{
pcmk__assert((fns != NULL) && (fns->rsc != NULL) && (fns->cluster != NULL)
&& (fns->pseudo != NULL) && (fns->fence != NULL));
crm_debug("Setting custom functions for executing transition graphs");
graph_fns = fns;
}
/*!
* \internal
* \brief Check whether a graph synapse is ready to be executed
*
* \param[in,out] graph Transition graph that synapse is part of
* \param[in,out] synapse Synapse to check
*
* \return true if synapse is ready, false otherwise
*/
static bool
should_fire_synapse(pcmk__graph_t *graph, pcmk__graph_synapse_t *synapse)
{
GList *lpc = NULL;
pcmk__set_synapse_flags(synapse, pcmk__synapse_ready);
for (lpc = synapse->inputs; lpc != NULL; lpc = lpc->next) {
pcmk__graph_action_t *prereq = (pcmk__graph_action_t *) lpc->data;
if (!(pcmk_is_set(prereq->flags, pcmk__graph_action_confirmed))) {
crm_trace("Input %d for synapse %d not yet confirmed",
prereq->id, synapse->id);
pcmk__clear_synapse_flags(synapse, pcmk__synapse_ready);
break;
} else if (pcmk_is_set(prereq->flags, pcmk__graph_action_failed)
&& !pcmk_is_set(prereq->flags,
pcmk__graph_action_can_fail)) {
crm_trace("Input %d for synapse %d confirmed but failed",
prereq->id, synapse->id);
pcmk__clear_synapse_flags(synapse, pcmk__synapse_ready);
break;
}
}
if (pcmk_is_set(synapse->flags, pcmk__synapse_ready)) {
crm_trace("Synapse %d is ready to execute", synapse->id);
} else {
return false;
}
for (lpc = synapse->actions; lpc != NULL; lpc = lpc->next) {
pcmk__graph_action_t *a = (pcmk__graph_action_t *) lpc->data;
if (a->type == pcmk__pseudo_graph_action) {
/* None of the below applies to pseudo ops */
} else if (synapse->priority < graph->abort_priority) {
crm_trace("Skipping synapse %d: priority %d is less than "
"abort priority %d",
synapse->id, synapse->priority, graph->abort_priority);
graph->skipped++;
return false;
} else if (graph_fns->allowed && !(graph_fns->allowed(graph, a))) {
crm_trace("Deferring synapse %d: not allowed", synapse->id);
return false;
}
}
return true;
}
/*!
* \internal
* \brief Initiate an action from a transition graph
*
* \param[in,out] graph Transition graph containing action
* \param[in,out] action Action to execute
*
* \return Standard Pacemaker return code
*/
static int
initiate_action(pcmk__graph_t *graph, pcmk__graph_action_t *action)
{
const char *id = pcmk__xe_id(action->xml);
CRM_CHECK(id != NULL, return EINVAL);
CRM_CHECK(!pcmk_is_set(action->flags, pcmk__graph_action_executed),
return pcmk_rc_already);
pcmk__set_graph_action_flags(action, pcmk__graph_action_executed);
switch (action->type) {
case pcmk__pseudo_graph_action:
crm_trace("Executing pseudo-action %d (%s)", action->id, id);
return graph_fns->pseudo(graph, action);
case pcmk__rsc_graph_action:
crm_trace("Executing resource action %d (%s)", action->id, id);
return graph_fns->rsc(graph, action);
case pcmk__cluster_graph_action:
if (pcmk__str_eq(crm_element_value(action->xml, PCMK_XA_OPERATION),
PCMK_ACTION_STONITH, pcmk__str_none)) {
crm_trace("Executing fencing action %d (%s)",
action->id, id);
return graph_fns->fence(graph, action);
}
crm_trace("Executing cluster action %d (%s)", action->id, id);
return graph_fns->cluster(graph, action);
default:
crm_err("Unsupported graph action type <%s " PCMK_XA_ID "='%s'> "
"(bug?)",
action->xml->name, id);
return EINVAL;
}
}
/*!
* \internal
* \brief Execute a graph synapse
*
* \param[in,out] graph Transition graph with synapse to execute
* \param[in,out] synapse Synapse to execute
*
* \return Standard Pacemaker return value
*/
static int
fire_synapse(pcmk__graph_t *graph, pcmk__graph_synapse_t *synapse)
{
pcmk__set_synapse_flags(synapse, pcmk__synapse_executed);
for (GList *lpc = synapse->actions; lpc != NULL; lpc = lpc->next) {
pcmk__graph_action_t *action = (pcmk__graph_action_t *) lpc->data;
int rc = initiate_action(graph, action);
if (rc != pcmk_rc_ok) {
crm_err("Failed initiating <%s " PCMK_XA_ID "=%d> in synapse %d: "
"%s",
action->xml->name, action->id, synapse->id,
pcmk_rc_str(rc));
pcmk__set_synapse_flags(synapse, pcmk__synapse_confirmed);
pcmk__set_graph_action_flags(action,
pcmk__graph_action_confirmed
|pcmk__graph_action_failed);
return pcmk_rc_error;
}
}
return pcmk_rc_ok;
}
/*!
* \internal
* \brief Dummy graph method that can be used with simulations
*
* \param[in,out] graph Transition graph containing action
* \param[in,out] action Graph action to be initiated
*
* \return Standard Pacemaker return code
* \note If the PE_fail environment variable is set to the action ID,
* then the graph action will be marked as failed.
*/
static int
pseudo_action_dummy(pcmk__graph_t *graph, pcmk__graph_action_t *action)
{
static int fail = -1;
if (fail < 0) {
long long fail_ll;
if ((pcmk__scan_ll(getenv("PE_fail"), &fail_ll, 0LL) == pcmk_rc_ok)
&& (fail_ll > 0LL) && (fail_ll <= INT_MAX)) {
fail = (int) fail_ll;
} else {
fail = 0;
}
}
if (action->id == fail) {
crm_err("Dummy event handler: pretending action %d failed", action->id);
pcmk__set_graph_action_flags(action, pcmk__graph_action_failed);
graph->abort_priority = PCMK_SCORE_INFINITY;
} else {
crm_trace("Dummy event handler: action %d initiated", action->id);
}
pcmk__set_graph_action_flags(action, pcmk__graph_action_confirmed);
pcmk__update_graph(graph, action);
return pcmk_rc_ok;
}
static pcmk__graph_functions_t default_fns = {
pseudo_action_dummy,
pseudo_action_dummy,
pseudo_action_dummy,
pseudo_action_dummy
};
/*!
* \internal
* \brief Execute all actions in a transition graph
*
* \param[in,out] graph Transition graph to execute
*
* \return Status of transition after execution
*/
enum pcmk__graph_status
pcmk__execute_graph(pcmk__graph_t *graph)
{
GList *lpc = NULL;
int log_level = LOG_DEBUG;
enum pcmk__graph_status pass_result = pcmk__graph_active;
const char *status = "In progress";
if (graph_fns == NULL) {
graph_fns = &default_fns;
}
if (graph == NULL) {
return pcmk__graph_complete;
}
graph->fired = 0;
graph->pending = 0;
graph->skipped = 0;
graph->completed = 0;
graph->incomplete = 0;
// Count completed and in-flight synapses
for (lpc = graph->synapses; lpc != NULL; lpc = lpc->next) {
pcmk__graph_synapse_t *synapse = (pcmk__graph_synapse_t *) lpc->data;
if (pcmk_is_set(synapse->flags, pcmk__synapse_confirmed)) {
graph->completed++;
} else if (!pcmk_is_set(synapse->flags, pcmk__synapse_failed)
&& pcmk_is_set(synapse->flags, pcmk__synapse_executed)) {
graph->pending++;
}
}
crm_trace("Executing graph %d (%d synapses already completed, %d pending)",
graph->id, graph->completed, graph->pending);
// Execute any synapses that are ready
for (lpc = graph->synapses; lpc != NULL; lpc = lpc->next) {
pcmk__graph_synapse_t *synapse = (pcmk__graph_synapse_t *) lpc->data;
if ((graph->batch_limit > 0)
&& (graph->pending >= graph->batch_limit)) {
crm_debug("Throttling graph execution: batch limit (%d) reached",
graph->batch_limit);
break;
} else if (pcmk_is_set(synapse->flags, pcmk__synapse_failed)) {
graph->skipped++;
continue;
} else if (pcmk_any_flags_set(synapse->flags,
pcmk__synapse_confirmed
|pcmk__synapse_executed)) {
continue; // Already handled
} else if (should_fire_synapse(graph, synapse)) {
graph->fired++;
if (fire_synapse(graph, synapse) != pcmk_rc_ok) {
crm_err("Synapse %d failed to fire", synapse->id);
log_level = LOG_ERR;
graph->abort_priority = PCMK_SCORE_INFINITY;
graph->incomplete++;
graph->fired--;
}
if (!(pcmk_is_set(synapse->flags, pcmk__synapse_confirmed))) {
graph->pending++;
}
} else {
crm_trace("Synapse %d cannot fire", synapse->id);
graph->incomplete++;
}
}
if ((graph->pending == 0) && (graph->fired == 0)) {
graph->complete = true;
if ((graph->incomplete != 0) && (graph->abort_priority <= 0)) {
log_level = LOG_WARNING;
pass_result = pcmk__graph_terminated;
status = "Terminated";
} else if (graph->skipped != 0) {
log_level = LOG_NOTICE;
pass_result = pcmk__graph_complete;
status = "Stopped";
} else {
log_level = LOG_NOTICE;
pass_result = pcmk__graph_complete;
status = "Complete";
}
} else if (graph->fired == 0) {
pass_result = pcmk__graph_pending;
}
do_crm_log(log_level,
"Transition %d (Complete=%d, Pending=%d,"
" Fired=%d, Skipped=%d, Incomplete=%d, Source=%s): %s",
graph->id, graph->completed, graph->pending, graph->fired,
graph->skipped, graph->incomplete, graph->source, status);
return pass_result;
}
/*
* Functions for unpacking transition graph XML into structs
*/
/*!
* \internal
* \brief Unpack a transition graph action from XML
*
* \param[in] parent Synapse that action is part of
* \param[in] xml_action Action XML to unparse
*
* \return Newly allocated action on success, or NULL otherwise
*/
static pcmk__graph_action_t *
unpack_action(pcmk__graph_synapse_t *parent, xmlNode *xml_action)
{
enum pcmk__graph_action_type action_type;
pcmk__graph_action_t *action = NULL;
const char *value = pcmk__xe_id(xml_action);
if (value == NULL) {
crm_err("Ignoring transition graph action without " PCMK_XA_ID
" (bug?)");
crm_log_xml_trace(xml_action, "invalid");
return NULL;
}
if (pcmk__xe_is(xml_action, PCMK__XE_RSC_OP)) {
action_type = pcmk__rsc_graph_action;
} else if (pcmk__xe_is(xml_action, PCMK__XE_PSEUDO_EVENT)) {
action_type = pcmk__pseudo_graph_action;
} else if (pcmk__xe_is(xml_action, PCMK__XE_CRM_EVENT)) {
action_type = pcmk__cluster_graph_action;
} else {
crm_err("Ignoring transition graph action of unknown type '%s' (bug?)",
xml_action->name);
crm_log_xml_trace(xml_action, "invalid");
return NULL;
}
action = calloc(1, sizeof(pcmk__graph_action_t));
if (action == NULL) {
crm_perror(LOG_CRIT, "Cannot unpack transition graph action");
crm_log_xml_trace(xml_action, "lost");
return NULL;
}
pcmk__scan_min_int(value, &(action->id), -1);
action->type = pcmk__rsc_graph_action;
action->xml = pcmk__xml_copy(NULL, xml_action);
action->synapse = parent;
action->type = action_type;
action->params = xml2list(action->xml);
value = crm_meta_value(action->params, PCMK_META_TIMEOUT);
pcmk__scan_min_int(value, &(action->timeout), 0);
/* Take PCMK_META_START_DELAY into account for the timeout of the action
* timer
*/
value = crm_meta_value(action->params, PCMK_META_START_DELAY);
{
int start_delay;
pcmk__scan_min_int(value, &start_delay, 0);
action->timeout += start_delay;
}
if (pcmk__guint_from_hash(action->params, CRM_META "_" PCMK_META_INTERVAL,
0, &(action->interval_ms)) != pcmk_rc_ok) {
action->interval_ms = 0;
}
value = crm_meta_value(action->params, PCMK__META_CAN_FAIL);
if (value != NULL) {
+ // @COMPAT Not possible with schema validation enabled
int can_fail = 0;
if ((crm_str_to_boolean(value, &can_fail) > 0) && (can_fail > 0)) {
pcmk__set_graph_action_flags(action, pcmk__graph_action_can_fail);
} else {
pcmk__clear_graph_action_flags(action, pcmk__graph_action_can_fail);
}
if (pcmk_is_set(action->flags, pcmk__graph_action_can_fail)) {
crm_warn("Support for the " PCMK__META_CAN_FAIL " meta-attribute "
"is deprecated and will be removed in a future release");
}
}
crm_trace("Action %d has timer set to %dms", action->id, action->timeout);
return action;
}
/*!
* \internal
* \brief Unpack transition graph synapse from XML
*
* \param[in,out] new_graph Transition graph that synapse is part of
* \param[in] xml_synapse Synapse XML
*
* \return Newly allocated synapse on success, or NULL otherwise
*/
static pcmk__graph_synapse_t *
unpack_synapse(pcmk__graph_t *new_graph, const xmlNode *xml_synapse)
{
const char *value = NULL;
xmlNode *action_set = NULL;
pcmk__graph_synapse_t *new_synapse = NULL;
crm_trace("Unpacking synapse %s", pcmk__xe_id(xml_synapse));
new_synapse = calloc(1, sizeof(pcmk__graph_synapse_t));
if (new_synapse == NULL) {
return NULL;
}
pcmk__scan_min_int(pcmk__xe_id(xml_synapse), &(new_synapse->id), 0);
value = crm_element_value(xml_synapse, PCMK__XA_PRIORITY);
pcmk__scan_min_int(value, &(new_synapse->priority), 0);
CRM_CHECK(new_synapse->id >= 0,
free_graph_synapse((gpointer) new_synapse); return NULL);
new_graph->num_synapses++;
crm_trace("Unpacking synapse %s action sets",
crm_element_value(xml_synapse, PCMK_XA_ID));
for (action_set = pcmk__xe_first_child(xml_synapse, "action_set", NULL,
NULL);
action_set != NULL; action_set = pcmk__xe_next_same(action_set)) {
for (xmlNode *action = pcmk__xe_first_child(action_set, NULL, NULL,
NULL);
action != NULL; action = pcmk__xe_next(action)) {
pcmk__graph_action_t *new_action = unpack_action(new_synapse,
action);
if (new_action == NULL) {
continue;
}
crm_trace("Adding action %d to synapse %d",
new_action->id, new_synapse->id);
new_graph->num_actions++;
new_synapse->actions = g_list_append(new_synapse->actions,
new_action);
}
}
crm_trace("Unpacking synapse %s inputs", pcmk__xe_id(xml_synapse));
for (xmlNode *inputs = pcmk__xe_first_child(xml_synapse, "inputs", NULL,
NULL);
inputs != NULL; inputs = pcmk__xe_next_same(inputs)) {
for (xmlNode *trigger = pcmk__xe_first_child(inputs, "trigger", NULL,
NULL);
trigger != NULL; trigger = pcmk__xe_next_same(trigger)) {
for (xmlNode *input = pcmk__xe_first_child(trigger, NULL, NULL,
NULL);
input != NULL; input = pcmk__xe_next(input)) {
pcmk__graph_action_t *new_input = unpack_action(new_synapse,
input);
if (new_input == NULL) {
continue;
}
crm_trace("Adding input %d to synapse %d",
new_input->id, new_synapse->id);
new_synapse->inputs = g_list_append(new_synapse->inputs,
new_input);
}
}
}
return new_synapse;
}
/*!
* \internal
* \brief Unpack transition graph XML
*
* \param[in] xml_graph Transition graph XML to unpack
* \param[in] reference Where the XML came from (for logging)
*
* \return Newly allocated transition graph on success, NULL otherwise
* \note The caller is responsible for freeing the return value using
* pcmk__free_graph().
* \note The XML is expected to be structured like:
...
...
*/
pcmk__graph_t *
pcmk__unpack_graph(const xmlNode *xml_graph, const char *reference)
{
pcmk__graph_t *new_graph = NULL;
new_graph = calloc(1, sizeof(pcmk__graph_t));
if (new_graph == NULL) {
return NULL;
}
new_graph->source = strdup(pcmk__s(reference, "unknown"));
if (new_graph->source == NULL) {
pcmk__free_graph(new_graph);
return NULL;
}
new_graph->completion_action = pcmk__graph_done;
// Parse top-level attributes from PCMK__XE_TRANSITION_GRAPH
if (xml_graph != NULL) {
const char *buf = crm_element_value(xml_graph, "transition_id");
CRM_CHECK(buf != NULL,
pcmk__free_graph(new_graph); return NULL);
pcmk__scan_min_int(buf, &(new_graph->id), 1);
buf = crm_element_value(xml_graph, PCMK_OPT_CLUSTER_DELAY);
CRM_CHECK(buf != NULL,
pcmk__free_graph(new_graph); return NULL);
pcmk_parse_interval_spec(buf, &(new_graph->network_delay));
buf = crm_element_value(xml_graph, PCMK_OPT_STONITH_TIMEOUT);
if (buf == NULL) {
new_graph->stonith_timeout = new_graph->network_delay;
} else {
pcmk_parse_interval_spec(buf, &(new_graph->stonith_timeout));
}
// Use 0 (dynamic limit) as default/invalid, -1 (no limit) as minimum
buf = crm_element_value(xml_graph, PCMK_OPT_BATCH_LIMIT);
if ((buf == NULL)
|| (pcmk__scan_min_int(buf, &(new_graph->batch_limit),
-1) != pcmk_rc_ok)) {
new_graph->batch_limit = 0;
}
buf = crm_element_value(xml_graph, PCMK_OPT_MIGRATION_LIMIT);
pcmk__scan_min_int(buf, &(new_graph->migration_limit), -1);
new_graph->failed_stop_offset =
crm_element_value_copy(xml_graph, "failed-stop-offset");
new_graph->failed_start_offset =
crm_element_value_copy(xml_graph, "failed-start-offset");
if (crm_element_value_epoch(xml_graph, "recheck-by",
&(new_graph->recheck_by)) != pcmk_ok) {
new_graph->recheck_by = 0;
}
}
// Unpack each child element
for (const xmlNode *synapse_xml = pcmk__xe_first_child(xml_graph,
"synapse", NULL,
NULL);
synapse_xml != NULL; synapse_xml = pcmk__xe_next_same(synapse_xml)) {
pcmk__graph_synapse_t *new_synapse = unpack_synapse(new_graph,
synapse_xml);
if (new_synapse != NULL) {
new_graph->synapses = g_list_append(new_graph->synapses,
new_synapse);
}
}
crm_debug("Unpacked transition %d from %s: %d actions in %d synapses",
new_graph->id, new_graph->source, new_graph->num_actions,
new_graph->num_synapses);
return new_graph;
}
/*
* Other transition graph utilities
*/
/*!
* \internal
* \brief Synthesize an executor event from a graph action
*
* \param[in] resource If not NULL, use greater call ID than in this XML
* \param[in] action Graph action
* \param[in] status What to use as event execution status
* \param[in] rc What to use as event exit status
* \param[in] exit_reason What to use as event exit reason
*
* \return Newly allocated executor event on success, or NULL otherwise
*/
lrmd_event_data_t *
pcmk__event_from_graph_action(const xmlNode *resource,
const pcmk__graph_action_t *action,
int status, int rc, const char *exit_reason)
{
lrmd_event_data_t *op = NULL;
GHashTableIter iter;
const char *name = NULL;
const char *value = NULL;
xmlNode *action_resource = NULL;
CRM_CHECK(action != NULL, return NULL);
CRM_CHECK(action->type == pcmk__rsc_graph_action, return NULL);
action_resource = pcmk__xe_first_child(action->xml, PCMK_XE_PRIMITIVE, NULL,
NULL);
CRM_CHECK(action_resource != NULL, crm_log_xml_warn(action->xml, "invalid");
return NULL);
op = lrmd_new_event(pcmk__xe_id(action_resource),
crm_element_value(action->xml, PCMK_XA_OPERATION),
action->interval_ms);
lrmd__set_result(op, rc, status, exit_reason);
op->t_run = time(NULL);
op->t_rcchange = op->t_run;
op->params = pcmk__strkey_table(free, free);
g_hash_table_iter_init(&iter, action->params);
while (g_hash_table_iter_next(&iter, (void **)&name, (void **)&value)) {
pcmk__insert_dup(op->params, name, value);
}
for (xmlNode *xop = pcmk__xe_first_child(resource, NULL, NULL, NULL);
xop != NULL; xop = pcmk__xe_next(xop)) {
int tmp = 0;
crm_element_value_int(xop, PCMK__XA_CALL_ID, &tmp);
crm_debug("Got call_id=%d for %s", tmp, pcmk__xe_id(resource));
if (tmp > op->call_id) {
op->call_id = tmp;
}
}
op->call_id++;
return op;
}
diff --git a/lib/pengine/complex.c b/lib/pengine/complex.c
index 08c871c3ea..148771ff08 100644
--- a/lib/pengine/complex.c
+++ b/lib/pengine/complex.c
@@ -1,1290 +1,1291 @@
/*
* Copyright 2004-2024 the Pacemaker project contributors
*
* The version control history for this file may have further details.
*
* This source code is licensed under the GNU Lesser General Public License
* version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
*/
#include
#include
#include
#include
#include
#include
#include "pe_status_private.h"
void populate_hash(xmlNode * nvpair_list, GHashTable * hash, const char **attrs, int attrs_length);
static pcmk_node_t *active_node(const pcmk_resource_t *rsc,
unsigned int *count_all,
unsigned int *count_clean);
static pcmk__rsc_methods_t resource_class_functions[] = {
{
native_unpack,
native_find_rsc,
native_parameter,
native_active,
native_resource_state,
native_location,
native_free,
pe__count_common,
pe__native_is_filtered,
active_node,
pe__primitive_max_per_node,
},
{
group_unpack,
native_find_rsc,
native_parameter,
group_active,
group_resource_state,
native_location,
group_free,
pe__count_common,
pe__group_is_filtered,
active_node,
pe__group_max_per_node,
},
{
clone_unpack,
native_find_rsc,
native_parameter,
clone_active,
clone_resource_state,
native_location,
clone_free,
pe__count_common,
pe__clone_is_filtered,
active_node,
pe__clone_max_per_node,
},
{
pe__unpack_bundle,
native_find_rsc,
native_parameter,
pe__bundle_active,
pe__bundle_resource_state,
native_location,
pe__free_bundle,
pe__count_bundle,
pe__bundle_is_filtered,
pe__bundle_active_node,
pe__bundle_max_per_node,
}
};
static enum pcmk__rsc_variant
get_resource_type(const char *name)
{
if (pcmk__str_eq(name, PCMK_XE_PRIMITIVE, pcmk__str_casei)) {
return pcmk__rsc_variant_primitive;
} else if (pcmk__str_eq(name, PCMK_XE_GROUP, pcmk__str_casei)) {
return pcmk__rsc_variant_group;
} else if (pcmk__str_eq(name, PCMK_XE_CLONE, pcmk__str_casei)) {
return pcmk__rsc_variant_clone;
} else if (pcmk__str_eq(name, PCMK_XE_BUNDLE, pcmk__str_casei)) {
return pcmk__rsc_variant_bundle;
}
return pcmk__rsc_variant_unknown;
}
/*!
* \internal
* \brief Insert a meta-attribute if not already present
*
* \param[in] key Meta-attribute name
* \param[in] value Meta-attribute value to add if not already present
* \param[in,out] table Meta-attribute hash table to insert into
*
* \note This is like pcmk__insert_meta() except it won't overwrite existing
* values.
*/
static void
dup_attr(gpointer key, gpointer value, gpointer user_data)
{
GHashTable *table = user_data;
CRM_CHECK((key != NULL) && (table != NULL), return);
if (pcmk__str_eq((const char *) value, "#default", pcmk__str_casei)) {
// @COMPAT Deprecated since 2.1.8
pcmk__config_warn("Support for setting meta-attributes (such as %s) to "
"the explicit value '#default' is deprecated and "
"will be removed in a future release",
(const char *) key);
} else if ((value != NULL) && (g_hash_table_lookup(table, key) == NULL)) {
pcmk__insert_dup(table, (const char *) key, (const char *) value);
}
}
static void
expand_parents_fixed_nvpairs(pcmk_resource_t *rsc,
pe_rule_eval_data_t *rule_data,
GHashTable *meta_hash, pcmk_scheduler_t *scheduler)
{
GHashTable *parent_orig_meta = pcmk__strkey_table(free, free);
pcmk_resource_t *p = rsc->priv->parent;
if (p == NULL) {
return ;
}
/* Search all parent resources, get the fixed value of
* PCMK_XE_META_ATTRIBUTES set only in the original xml, and stack it in the
* hash table. The fixed value of the lower parent resource takes precedence
* and is not overwritten.
*/
while(p != NULL) {
/* A hash table for comparison is generated, including the id-ref. */
pe__unpack_dataset_nvpairs(p->priv->xml, PCMK_XE_META_ATTRIBUTES,
rule_data, parent_orig_meta, NULL,
scheduler);
p = p->priv->parent;
}
if (parent_orig_meta != NULL) {
// This will not overwrite any values already existing for child
g_hash_table_foreach(parent_orig_meta, dup_attr, meta_hash);
}
if (parent_orig_meta != NULL) {
g_hash_table_destroy(parent_orig_meta);
}
return ;
}
/*
* \brief Get fully evaluated resource meta-attributes
*
* \param[in,out] meta_hash Where to store evaluated meta-attributes
* \param[in] rsc Resource to get meta-attributes for
* \param[in] node Ignored
* \param[in,out] scheduler Scheduler data
*/
void
get_meta_attributes(GHashTable * meta_hash, pcmk_resource_t * rsc,
pcmk_node_t *node, pcmk_scheduler_t *scheduler)
{
pe_rsc_eval_data_t rsc_rule_data = {
.standard = crm_element_value(rsc->priv->xml, PCMK_XA_CLASS),
.provider = crm_element_value(rsc->priv->xml, PCMK_XA_PROVIDER),
.agent = crm_element_value(rsc->priv->xml, PCMK_XA_TYPE)
};
pe_rule_eval_data_t rule_data = {
.node_hash = NULL,
.now = scheduler->priv->now,
.match_data = NULL,
.rsc_data = &rsc_rule_data,
.op_data = NULL
};
for (xmlAttrPtr a = pcmk__xe_first_attr(rsc->priv->xml);
a != NULL; a = a->next) {
if (a->children != NULL) {
dup_attr((gpointer) a->name, (gpointer) a->children->content,
meta_hash);
}
}
pe__unpack_dataset_nvpairs(rsc->priv->xml, PCMK_XE_META_ATTRIBUTES,
&rule_data, meta_hash, NULL, scheduler);
/* Set the PCMK_XE_META_ATTRIBUTES explicitly set in the parent resource to
* the hash table of the child resource. If it is already explicitly set as
* a child, it will not be overwritten.
*/
if (rsc->priv->parent != NULL) {
expand_parents_fixed_nvpairs(rsc, &rule_data, meta_hash, scheduler);
}
/* check the defaults */
pe__unpack_dataset_nvpairs(scheduler->priv->rsc_defaults,
PCMK_XE_META_ATTRIBUTES, &rule_data, meta_hash,
NULL, scheduler);
/* If there is PCMK_XE_META_ATTRIBUTES that the parent resource has not
* explicitly set, set a value that is not set from PCMK_XE_RSC_DEFAULTS
* either. The values already set up to this point will not be overwritten.
*/
if (rsc->priv->parent != NULL) {
g_hash_table_foreach(rsc->priv->parent->priv->meta, dup_attr,
meta_hash);
}
}
/*!
* \brief Get final values of a resource's instance attributes
*
* \param[in,out] instance_attrs Where to store the instance attributes
* \param[in] rsc Resource to get instance attributes for
* \param[in] node If not NULL, evaluate rules for this node
* \param[in,out] scheduler Scheduler data
*/
void
get_rsc_attributes(GHashTable *instance_attrs, const pcmk_resource_t *rsc,
const pcmk_node_t *node, pcmk_scheduler_t *scheduler)
{
pe_rule_eval_data_t rule_data = {
.node_hash = NULL,
.now = NULL,
.match_data = NULL,
.rsc_data = NULL,
.op_data = NULL
};
CRM_CHECK((instance_attrs != NULL) && (rsc != NULL) && (scheduler != NULL),
return);
rule_data.now = scheduler->priv->now;
if (node != NULL) {
rule_data.node_hash = node->priv->attrs;
}
// Evaluate resource's own values, then its ancestors' values
pe__unpack_dataset_nvpairs(rsc->priv->xml, PCMK_XE_INSTANCE_ATTRIBUTES,
&rule_data, instance_attrs, NULL, scheduler);
if (rsc->priv->parent != NULL) {
get_rsc_attributes(instance_attrs, rsc->priv->parent, node, scheduler);
}
}
static char *
template_op_key(xmlNode * op)
{
const char *name = crm_element_value(op, PCMK_XA_NAME);
const char *role = crm_element_value(op, PCMK_XA_ROLE);
char *key = NULL;
if ((role == NULL)
|| pcmk__strcase_any_of(role, PCMK_ROLE_STARTED, PCMK_ROLE_UNPROMOTED,
PCMK__ROLE_UNPROMOTED_LEGACY, NULL)) {
role = PCMK__ROLE_UNKNOWN;
}
key = crm_strdup_printf("%s-%s", name, role);
return key;
}
static gboolean
unpack_template(xmlNode *xml_obj, xmlNode **expanded_xml,
pcmk_scheduler_t *scheduler)
{
xmlNode *cib_resources = NULL;
xmlNode *template = NULL;
xmlNode *new_xml = NULL;
xmlNode *child_xml = NULL;
xmlNode *rsc_ops = NULL;
xmlNode *template_ops = NULL;
const char *template_ref = NULL;
const char *id = NULL;
if (xml_obj == NULL) {
pcmk__config_err("No resource object for template unpacking");
return FALSE;
}
template_ref = crm_element_value(xml_obj, PCMK_XA_TEMPLATE);
if (template_ref == NULL) {
return TRUE;
}
id = pcmk__xe_id(xml_obj);
if (id == NULL) {
pcmk__config_err("'%s' object must have a id", xml_obj->name);
return FALSE;
}
if (pcmk__str_eq(template_ref, id, pcmk__str_none)) {
pcmk__config_err("The resource object '%s' should not reference itself",
id);
return FALSE;
}
cib_resources = get_xpath_object("//" PCMK_XE_RESOURCES, scheduler->input,
LOG_TRACE);
if (cib_resources == NULL) {
pcmk__config_err("No resources configured");
return FALSE;
}
template = pcmk__xe_first_child(cib_resources, PCMK_XE_TEMPLATE,
PCMK_XA_ID, template_ref);
if (template == NULL) {
pcmk__config_err("No template named '%s'", template_ref);
return FALSE;
}
new_xml = pcmk__xml_copy(NULL, template);
xmlNodeSetName(new_xml, xml_obj->name);
crm_xml_add(new_xml, PCMK_XA_ID, id);
crm_xml_add(new_xml, PCMK__META_CLONE,
crm_element_value(xml_obj, PCMK__META_CLONE));
template_ops = pcmk__xe_first_child(new_xml, PCMK_XE_OPERATIONS, NULL,
NULL);
for (child_xml = pcmk__xe_first_child(xml_obj, NULL, NULL, NULL);
child_xml != NULL; child_xml = pcmk__xe_next(child_xml)) {
xmlNode *new_child = pcmk__xml_copy(new_xml, child_xml);
if (pcmk__xe_is(new_child, PCMK_XE_OPERATIONS)) {
rsc_ops = new_child;
}
}
if (template_ops && rsc_ops) {
xmlNode *op = NULL;
GHashTable *rsc_ops_hash = pcmk__strkey_table(free, NULL);
for (op = pcmk__xe_first_child(rsc_ops, NULL, NULL, NULL); op != NULL;
op = pcmk__xe_next(op)) {
char *key = template_op_key(op);
g_hash_table_insert(rsc_ops_hash, key, op);
}
for (op = pcmk__xe_first_child(template_ops, NULL, NULL, NULL);
op != NULL; op = pcmk__xe_next(op)) {
char *key = template_op_key(op);
if (g_hash_table_lookup(rsc_ops_hash, key) == NULL) {
pcmk__xml_copy(rsc_ops, op);
}
free(key);
}
if (rsc_ops_hash) {
g_hash_table_destroy(rsc_ops_hash);
}
pcmk__xml_free(template_ops);
}
/*pcmk__xml_free(*expanded_xml); */
*expanded_xml = new_xml;
#if 0 /* Disable multi-level templates for now */
if (!unpack_template(new_xml, expanded_xml, scheduler)) {
pcmk__xml_free(*expanded_xml);
*expanded_xml = NULL;
return FALSE;
}
#endif
return TRUE;
}
static gboolean
add_template_rsc(xmlNode *xml_obj, pcmk_scheduler_t *scheduler)
{
const char *template_ref = NULL;
const char *id = NULL;
if (xml_obj == NULL) {
pcmk__config_err("No resource object for processing resource list "
"of template");
return FALSE;
}
template_ref = crm_element_value(xml_obj, PCMK_XA_TEMPLATE);
if (template_ref == NULL) {
return TRUE;
}
id = pcmk__xe_id(xml_obj);
if (id == NULL) {
pcmk__config_err("'%s' object must have a id", xml_obj->name);
return FALSE;
}
if (pcmk__str_eq(template_ref, id, pcmk__str_none)) {
pcmk__config_err("The resource object '%s' should not reference itself",
id);
return FALSE;
}
pcmk__add_idref(scheduler->priv->templates, template_ref, id);
return TRUE;
}
/*!
* \internal
* \brief Check whether a clone or instance being unpacked is globally unique
*
* \param[in] rsc Clone or clone instance to check
*
* \return \c true if \p rsc is globally unique according to its
* meta-attributes, otherwise \c false
*/
static bool
detect_unique(const pcmk_resource_t *rsc)
{
const char *value = g_hash_table_lookup(rsc->priv->meta,
PCMK_META_GLOBALLY_UNIQUE);
if (value == NULL) { // Default to true if clone-node-max > 1
value = g_hash_table_lookup(rsc->priv->meta,
PCMK_META_CLONE_NODE_MAX);
if (value != NULL) {
int node_max = 1;
if ((pcmk__scan_min_int(value, &node_max, 0) == pcmk_rc_ok)
&& (node_max > 1)) {
return true;
}
}
return false;
}
return crm_is_true(value);
}
static void
free_params_table(gpointer data)
{
g_hash_table_destroy((GHashTable *) data);
}
/*!
* \brief Get a table of resource parameters
*
* \param[in,out] rsc Resource to query
* \param[in] node Node for evaluating rules (NULL for defaults)
* \param[in,out] scheduler Scheduler data
*
* \return Hash table containing resource parameter names and values
* (or NULL if \p rsc or \p scheduler is NULL)
* \note The returned table will be destroyed when the resource is freed, so
* callers should not destroy it.
*/
GHashTable *
pe_rsc_params(pcmk_resource_t *rsc, const pcmk_node_t *node,
pcmk_scheduler_t *scheduler)
{
GHashTable *params_on_node = NULL;
/* A NULL node is used to request the resource's default parameters
* (not evaluated for node), but we always want something non-NULL
* as a hash table key.
*/
const char *node_name = "";
// Sanity check
if ((rsc == NULL) || (scheduler == NULL)) {
return NULL;
}
if ((node != NULL) && (node->priv->name != NULL)) {
node_name = node->priv->name;
}
// Find the parameter table for given node
if (rsc->priv->parameter_cache == NULL) {
rsc->priv->parameter_cache = pcmk__strikey_table(free,
free_params_table);
} else {
params_on_node = g_hash_table_lookup(rsc->priv->parameter_cache,
node_name);
}
// If none exists yet, create one with parameters evaluated for node
if (params_on_node == NULL) {
params_on_node = pcmk__strkey_table(free, free);
get_rsc_attributes(params_on_node, rsc, node, scheduler);
g_hash_table_insert(rsc->priv->parameter_cache, strdup(node_name),
params_on_node);
}
return params_on_node;
}
/*!
* \internal
* \brief Unpack a resource's \c PCMK_META_REQUIRES meta-attribute
*
* \param[in,out] rsc Resource being unpacked
* \param[in] value Value of \c PCMK_META_REQUIRES meta-attribute
* \param[in] is_default Whether \p value was selected by default
*/
static void
unpack_requires(pcmk_resource_t *rsc, const char *value, bool is_default)
{
const pcmk_scheduler_t *scheduler = rsc->priv->scheduler;
if (pcmk__str_eq(value, PCMK_VALUE_NOTHING, pcmk__str_casei)) {
} else if (pcmk__str_eq(value, PCMK_VALUE_QUORUM, pcmk__str_casei)) {
pcmk__set_rsc_flags(rsc, pcmk__rsc_needs_quorum);
} else if (pcmk__str_eq(value, PCMK_VALUE_FENCING, pcmk__str_casei)) {
pcmk__set_rsc_flags(rsc, pcmk__rsc_needs_fencing);
if (!pcmk_is_set(scheduler->flags, pcmk__sched_fencing_enabled)) {
pcmk__config_warn("%s requires fencing but fencing is disabled",
rsc->id);
}
} else if (pcmk__str_eq(value, PCMK_VALUE_UNFENCING, pcmk__str_casei)) {
if (pcmk_is_set(rsc->flags, pcmk__rsc_fence_device)) {
pcmk__config_warn("Resetting \"" PCMK_META_REQUIRES "\" for %s "
"to \"" PCMK_VALUE_QUORUM "\" because fencing "
"devices cannot require unfencing", rsc->id);
unpack_requires(rsc, PCMK_VALUE_QUORUM, true);
return;
} else if (!pcmk_is_set(scheduler->flags, pcmk__sched_fencing_enabled)) {
pcmk__config_warn("Resetting \"" PCMK_META_REQUIRES "\" for %s "
"to \"" PCMK_VALUE_QUORUM "\" because fencing is "
"disabled", rsc->id);
unpack_requires(rsc, PCMK_VALUE_QUORUM, true);
return;
} else {
pcmk__set_rsc_flags(rsc, pcmk__rsc_needs_fencing
|pcmk__rsc_needs_unfencing);
}
} else {
const char *orig_value = value;
if (pcmk_is_set(rsc->flags, pcmk__rsc_fence_device)) {
value = PCMK_VALUE_QUORUM;
} else if (pcmk__is_primitive(rsc)
&& xml_contains_remote_node(rsc->priv->xml)) {
value = PCMK_VALUE_QUORUM;
} else if (pcmk_is_set(scheduler->flags, pcmk__sched_enable_unfencing)) {
value = PCMK_VALUE_UNFENCING;
} else if (pcmk_is_set(scheduler->flags, pcmk__sched_fencing_enabled)) {
value = PCMK_VALUE_FENCING;
} else if (scheduler->no_quorum_policy == pcmk_no_quorum_ignore) {
value = PCMK_VALUE_NOTHING;
} else {
value = PCMK_VALUE_QUORUM;
}
if (orig_value != NULL) {
pcmk__config_err("Resetting '" PCMK_META_REQUIRES "' for %s "
"to '%s' because '%s' is not valid",
rsc->id, value, orig_value);
}
unpack_requires(rsc, value, true);
return;
}
pcmk__rsc_trace(rsc, "\tRequired to start: %s%s", value,
(is_default? " (default)" : ""));
}
/*!
* \internal
* \brief Parse resource priority from meta-attribute
*
* \param[in,out] rsc Resource being unpacked
*/
static void
unpack_priority(pcmk_resource_t *rsc)
{
const char *value = g_hash_table_lookup(rsc->priv->meta,
PCMK_META_PRIORITY);
int rc = pcmk_parse_score(value, &(rsc->priv->priority), 0);
if (rc != pcmk_rc_ok) {
pcmk__config_warn("Using default (0) for resource %s "
PCMK_META_PRIORITY
" because '%s' is not a valid value: %s",
rsc->id, value, pcmk_rc_str(rc));
}
}
/*!
* \internal
* \brief Parse resource stickiness from meta-attribute
*
* \param[in,out] rsc Resource being unpacked
*/
static void
unpack_stickiness(pcmk_resource_t *rsc)
{
const char *value = g_hash_table_lookup(rsc->priv->meta,
PCMK_META_RESOURCE_STICKINESS);
if (pcmk__str_eq(value, PCMK_VALUE_DEFAULT, pcmk__str_casei)) {
// @COMPAT Deprecated since 2.1.8
pcmk__config_warn("Support for setting "
PCMK_META_RESOURCE_STICKINESS
" to the explicit value '" PCMK_VALUE_DEFAULT
"' is deprecated and will be removed in a "
"future release (just leave it unset)");
} else {
int rc = pcmk_parse_score(value, &(rsc->priv->stickiness), 0);
if (rc != pcmk_rc_ok) {
pcmk__config_warn("Using default (0) for resource %s "
PCMK_META_RESOURCE_STICKINESS
" because '%s' is not a valid value: %s",
rsc->id, value, pcmk_rc_str(rc));
}
}
}
/*!
* \internal
* \brief Parse resource migration threshold from meta-attribute
*
* \param[in,out] rsc Resource being unpacked
*/
static void
unpack_migration_threshold(pcmk_resource_t *rsc)
{
const char *value = g_hash_table_lookup(rsc->priv->meta,
PCMK_META_MIGRATION_THRESHOLD);
if (pcmk__str_eq(value, PCMK_VALUE_DEFAULT, pcmk__str_casei)) {
// @COMPAT Deprecated since 2.1.8
pcmk__config_warn("Support for setting "
PCMK_META_MIGRATION_THRESHOLD
" to the explicit value '" PCMK_VALUE_DEFAULT
"' is deprecated and will be removed in a "
"future release (just leave it unset)");
rsc->priv->ban_after_failures = PCMK_SCORE_INFINITY;
} else {
int rc = pcmk_parse_score(value, &(rsc->priv->ban_after_failures),
PCMK_SCORE_INFINITY);
if ((rc != pcmk_rc_ok) || (rsc->priv->ban_after_failures < 0)) {
pcmk__config_warn("Using default (" PCMK_VALUE_INFINITY
") for resource %s meta-attribute "
PCMK_META_MIGRATION_THRESHOLD
" because '%s' is not a valid value: %s",
rsc->id, value, pcmk_rc_str(rc));
rsc->priv->ban_after_failures = PCMK_SCORE_INFINITY;
}
}
}
/*!
* \internal
* \brief Unpack configuration XML for a given resource
*
* Unpack the XML object containing a resource's configuration into a new
* \c pcmk_resource_t object.
*
* \param[in] xml_obj XML node containing the resource's configuration
* \param[out] rsc Where to store the unpacked resource information
* \param[in] parent Resource's parent, if any
* \param[in,out] scheduler Scheduler data
*
* \return Standard Pacemaker return code
* \note If pcmk_rc_ok is returned, \p *rsc is guaranteed to be non-NULL, and
* the caller is responsible for freeing it using its variant-specific
* free() method. Otherwise, \p *rsc is guaranteed to be NULL.
*/
int
pe__unpack_resource(xmlNode *xml_obj, pcmk_resource_t **rsc,
pcmk_resource_t *parent, pcmk_scheduler_t *scheduler)
{
xmlNode *expanded_xml = NULL;
xmlNode *ops = NULL;
const char *value = NULL;
const char *id = NULL;
bool guest_node = false;
bool remote_node = false;
pcmk__resource_private_t *rsc_private = NULL;
pe_rule_eval_data_t rule_data = {
.node_hash = NULL,
.now = NULL,
.match_data = NULL,
.rsc_data = NULL,
.op_data = NULL
};
CRM_CHECK(rsc != NULL, return EINVAL);
CRM_CHECK((xml_obj != NULL) && (scheduler != NULL),
*rsc = NULL;
return EINVAL);
rule_data.now = scheduler->priv->now;
crm_log_xml_trace(xml_obj, "[raw XML]");
id = crm_element_value(xml_obj, PCMK_XA_ID);
if (id == NULL) {
pcmk__config_err("Ignoring <%s> configuration without " PCMK_XA_ID,
xml_obj->name);
return pcmk_rc_unpack_error;
}
if (unpack_template(xml_obj, &expanded_xml, scheduler) == FALSE) {
return pcmk_rc_unpack_error;
}
*rsc = calloc(1, sizeof(pcmk_resource_t));
if (*rsc == NULL) {
pcmk__sched_err(scheduler,
"Unable to allocate memory for resource '%s'", id);
return ENOMEM;
}
(*rsc)->priv = calloc(1, sizeof(pcmk__resource_private_t));
if ((*rsc)->priv == NULL) {
pcmk__sched_err(scheduler,
"Unable to allocate memory for resource '%s'", id);
free(*rsc);
return ENOMEM;
}
rsc_private = (*rsc)->priv;
rsc_private->scheduler = scheduler;
if (expanded_xml) {
crm_log_xml_trace(expanded_xml, "[expanded XML]");
rsc_private->xml = expanded_xml;
rsc_private->orig_xml = xml_obj;
} else {
rsc_private->xml = xml_obj;
rsc_private->orig_xml = NULL;
}
/* Do not use xml_obj from here on, use (*rsc)->xml in case templates are involved */
rsc_private->parent = parent;
ops = pcmk__xe_first_child(rsc_private->xml, PCMK_XE_OPERATIONS, NULL,
NULL);
rsc_private->ops_xml = pcmk__xe_resolve_idref(ops, scheduler->input);
rsc_private->variant = get_resource_type((const char *)
rsc_private->xml->name);
if (rsc_private->variant == pcmk__rsc_variant_unknown) {
pcmk__config_err("Ignoring resource '%s' of unknown type '%s'",
id, rsc_private->xml->name);
common_free(*rsc);
*rsc = NULL;
return pcmk_rc_unpack_error;
}
rsc_private->meta = pcmk__strkey_table(free, free);
rsc_private->utilization = pcmk__strkey_table(free, free);
rsc_private->probed_nodes = pcmk__strkey_table(NULL, free);
rsc_private->allowed_nodes = pcmk__strkey_table(NULL, free);
value = crm_element_value(rsc_private->xml, PCMK__META_CLONE);
if (value) {
(*rsc)->id = crm_strdup_printf("%s:%s", id, value);
pcmk__insert_meta(rsc_private, PCMK__META_CLONE, value);
} else {
(*rsc)->id = strdup(id);
}
rsc_private->fns = &resource_class_functions[rsc_private->variant];
get_meta_attributes(rsc_private->meta, *rsc, NULL, scheduler);
(*rsc)->flags = 0;
pcmk__set_rsc_flags(*rsc, pcmk__rsc_unassigned);
if (!pcmk_is_set(scheduler->flags, pcmk__sched_in_maintenance)) {
pcmk__set_rsc_flags(*rsc, pcmk__rsc_managed);
}
rsc_private->orig_role = pcmk_role_stopped;
rsc_private->next_role = pcmk_role_unknown;
unpack_priority(*rsc);
value = g_hash_table_lookup(rsc_private->meta, PCMK_META_CRITICAL);
if ((value == NULL) || crm_is_true(value)) {
pcmk__set_rsc_flags(*rsc, pcmk__rsc_critical);
}
value = g_hash_table_lookup(rsc_private->meta, PCMK_META_NOTIFY);
if (crm_is_true(value)) {
pcmk__set_rsc_flags(*rsc, pcmk__rsc_notify);
}
if (xml_contains_remote_node(rsc_private->xml)) {
pcmk__set_rsc_flags(*rsc, pcmk__rsc_is_remote_connection);
if (g_hash_table_lookup(rsc_private->meta, PCMK__META_CONTAINER)) {
guest_node = true;
} else {
remote_node = true;
}
}
value = g_hash_table_lookup(rsc_private->meta, PCMK_META_ALLOW_MIGRATE);
if (crm_is_true(value)) {
pcmk__set_rsc_flags(*rsc, pcmk__rsc_migratable);
} else if ((value == NULL) && remote_node) {
/* By default, we want remote nodes to be able
* to float around the cluster without having to stop all the
* resources within the remote-node before moving. Allowing
* migration support enables this feature. If this ever causes
* problems, migration support can be explicitly turned off with
* PCMK_META_ALLOW_MIGRATE=false.
*/
pcmk__set_rsc_flags(*rsc, pcmk__rsc_migratable);
}
value = g_hash_table_lookup(rsc_private->meta, PCMK_META_IS_MANAGED);
if (value != NULL) {
if (pcmk__str_eq(PCMK_VALUE_DEFAULT, value, pcmk__str_casei)) {
// @COMPAT Deprecated since 2.1.8
pcmk__config_warn("Support for setting " PCMK_META_IS_MANAGED
" to the explicit value '" PCMK_VALUE_DEFAULT
"' is deprecated and will be removed in a "
"future release (just leave it unset)");
} else if (crm_is_true(value)) {
pcmk__set_rsc_flags(*rsc, pcmk__rsc_managed);
} else {
pcmk__clear_rsc_flags(*rsc, pcmk__rsc_managed);
}
}
value = g_hash_table_lookup(rsc_private->meta, PCMK_META_MAINTENANCE);
if (crm_is_true(value)) {
pcmk__clear_rsc_flags(*rsc, pcmk__rsc_managed);
pcmk__set_rsc_flags(*rsc, pcmk__rsc_maintenance);
}
if (pcmk_is_set(scheduler->flags, pcmk__sched_in_maintenance)) {
pcmk__clear_rsc_flags(*rsc, pcmk__rsc_managed);
pcmk__set_rsc_flags(*rsc, pcmk__rsc_maintenance);
}
if (pcmk__is_clone(pe__const_top_resource(*rsc, false))) {
if (detect_unique(*rsc)) {
pcmk__set_rsc_flags(*rsc, pcmk__rsc_unique);
}
if (crm_is_true(g_hash_table_lookup((*rsc)->priv->meta,
PCMK_META_PROMOTABLE))) {
pcmk__set_rsc_flags(*rsc, pcmk__rsc_promotable);
}
} else {
pcmk__set_rsc_flags(*rsc, pcmk__rsc_unique);
}
// @COMPAT Deprecated meta-attribute
value = g_hash_table_lookup(rsc_private->meta, PCMK__META_RESTART_TYPE);
if (pcmk__str_eq(value, PCMK_VALUE_RESTART, pcmk__str_casei)) {
+ // @COMPAT Not possible with schema validation enabled
rsc_private->restart_type = pcmk__restart_restart;
pcmk__rsc_trace(*rsc, "%s dependency restart handling: restart",
(*rsc)->id);
pcmk__warn_once(pcmk__wo_restart_type,
"Support for " PCMK__META_RESTART_TYPE " is deprecated "
"and will be removed in a future release");
} else {
rsc_private->restart_type = pcmk__restart_ignore;
pcmk__rsc_trace(*rsc, "%s dependency restart handling: ignore",
(*rsc)->id);
}
value = g_hash_table_lookup(rsc_private->meta, PCMK_META_MULTIPLE_ACTIVE);
if (pcmk__str_eq(value, PCMK_VALUE_STOP_ONLY, pcmk__str_casei)) {
rsc_private->multiply_active_policy = pcmk__multiply_active_stop;
pcmk__rsc_trace(*rsc, "%s multiple running resource recovery: stop only",
(*rsc)->id);
} else if (pcmk__str_eq(value, PCMK_VALUE_BLOCK, pcmk__str_casei)) {
rsc_private->multiply_active_policy = pcmk__multiply_active_block;
pcmk__rsc_trace(*rsc, "%s multiple running resource recovery: block",
(*rsc)->id);
} else if (pcmk__str_eq(value, PCMK_VALUE_STOP_UNEXPECTED,
pcmk__str_casei)) {
rsc_private->multiply_active_policy = pcmk__multiply_active_unexpected;
pcmk__rsc_trace(*rsc,
"%s multiple running resource recovery: "
"stop unexpected instances",
(*rsc)->id);
} else { // PCMK_VALUE_STOP_START
if (!pcmk__str_eq(value, PCMK_VALUE_STOP_START,
pcmk__str_casei|pcmk__str_null_matches)) {
pcmk__config_warn("%s is not a valid value for "
PCMK_META_MULTIPLE_ACTIVE
", using default of "
"\"" PCMK_VALUE_STOP_START "\"",
value);
}
rsc_private->multiply_active_policy = pcmk__multiply_active_restart;
pcmk__rsc_trace(*rsc,
"%s multiple running resource recovery: stop/start",
(*rsc)->id);
}
unpack_stickiness(*rsc);
unpack_migration_threshold(*rsc);
if (pcmk__str_eq(crm_element_value(rsc_private->xml, PCMK_XA_CLASS),
PCMK_RESOURCE_CLASS_STONITH, pcmk__str_casei)) {
pcmk__set_scheduler_flags(scheduler, pcmk__sched_have_fencing);
pcmk__set_rsc_flags(*rsc, pcmk__rsc_fence_device);
}
value = g_hash_table_lookup(rsc_private->meta, PCMK_META_REQUIRES);
unpack_requires(*rsc, value, false);
value = g_hash_table_lookup(rsc_private->meta, PCMK_META_FAILURE_TIMEOUT);
if (value != NULL) {
pcmk_parse_interval_spec(value, &(rsc_private->failure_expiration_ms));
}
if (remote_node) {
GHashTable *params = pe_rsc_params(*rsc, NULL, scheduler);
/* Grabbing the value now means that any rules based on node attributes
* will evaluate to false, so such rules should not be used with
* PCMK_REMOTE_RA_RECONNECT_INTERVAL.
*
* @TODO Evaluate per node before using
*/
value = g_hash_table_lookup(params, PCMK_REMOTE_RA_RECONNECT_INTERVAL);
if (value) {
/* reconnect delay works by setting failure_timeout and preventing the
* connection from starting until the failure is cleared. */
pcmk_parse_interval_spec(value,
&(rsc_private->remote_reconnect_ms));
/* We want to override any default failure_timeout in use when remote
* PCMK_REMOTE_RA_RECONNECT_INTERVAL is in use.
*/
rsc_private->failure_expiration_ms =
rsc_private->remote_reconnect_ms;
}
}
get_target_role(*rsc, &(rsc_private->next_role));
pcmk__rsc_trace(*rsc, "%s desired next state: %s", (*rsc)->id,
(rsc_private->next_role == pcmk_role_unknown)?
"default" : pcmk_role_text(rsc_private->next_role));
if (rsc_private->fns->unpack(*rsc, scheduler) == FALSE) {
rsc_private->fns->free(*rsc);
*rsc = NULL;
return pcmk_rc_unpack_error;
}
if (pcmk_is_set(scheduler->flags, pcmk__sched_symmetric_cluster)) {
// This tag must stay exactly the same because it is tested elsewhere
resource_location(*rsc, NULL, 0, "symmetric_default", scheduler);
} else if (guest_node) {
/* remote resources tied to a container resource must always be allowed
* to opt-in to the cluster. Whether the connection resource is actually
* allowed to be placed on a node is dependent on the container resource */
resource_location(*rsc, NULL, 0, "remote_connection_default",
scheduler);
}
pcmk__rsc_trace(*rsc, "%s action notification: %s", (*rsc)->id,
pcmk_is_set((*rsc)->flags, pcmk__rsc_notify)? "required" : "not required");
pe__unpack_dataset_nvpairs(rsc_private->xml, PCMK_XE_UTILIZATION,
&rule_data, rsc_private->utilization, NULL,
scheduler);
if (expanded_xml) {
if (add_template_rsc(xml_obj, scheduler) == FALSE) {
rsc_private->fns->free(*rsc);
*rsc = NULL;
return pcmk_rc_unpack_error;
}
}
return pcmk_rc_ok;
}
gboolean
is_parent(pcmk_resource_t *child, pcmk_resource_t *rsc)
{
pcmk_resource_t *parent = child;
if (parent == NULL || rsc == NULL) {
return FALSE;
}
while (parent->priv->parent != NULL) {
if (parent->priv->parent == rsc) {
return TRUE;
}
parent = parent->priv->parent;
}
return FALSE;
}
pcmk_resource_t *
uber_parent(pcmk_resource_t *rsc)
{
pcmk_resource_t *parent = rsc;
if (parent == NULL) {
return NULL;
}
while ((parent->priv->parent != NULL)
&& !pcmk__is_bundle(parent->priv->parent)) {
parent = parent->priv->parent;
}
return parent;
}
/*!
* \internal
* \brief Get the topmost parent of a resource as a const pointer
*
* \param[in] rsc Resource to check
* \param[in] include_bundle If true, go all the way to bundle
*
* \return \p NULL if \p rsc is NULL, \p rsc if \p rsc has no parent,
* the bundle if \p rsc is bundled and \p include_bundle is true,
* otherwise the topmost parent of \p rsc up to a clone
*/
const pcmk_resource_t *
pe__const_top_resource(const pcmk_resource_t *rsc, bool include_bundle)
{
const pcmk_resource_t *parent = rsc;
if (parent == NULL) {
return NULL;
}
while (parent->priv->parent != NULL) {
if (!include_bundle && pcmk__is_bundle(parent->priv->parent)) {
break;
}
parent = parent->priv->parent;
}
return parent;
}
void
common_free(pcmk_resource_t * rsc)
{
if (rsc == NULL) {
return;
}
pcmk__rsc_trace(rsc, "Freeing %s", rsc->id);
if (rsc->priv->parameter_cache != NULL) {
g_hash_table_destroy(rsc->priv->parameter_cache);
}
if ((rsc->priv->parent == NULL)
&& pcmk_is_set(rsc->flags, pcmk__rsc_removed)) {
pcmk__xml_free(rsc->priv->xml);
rsc->priv->xml = NULL;
pcmk__xml_free(rsc->priv->orig_xml);
rsc->priv->orig_xml = NULL;
} else if (rsc->priv->orig_xml != NULL) {
// rsc->private->xml was expanded from a template
pcmk__xml_free(rsc->priv->xml);
rsc->priv->xml = NULL;
}
free(rsc->id);
free(rsc->priv->variant_opaque);
free(rsc->priv->history_id);
free(rsc->priv->pending_action);
free(rsc->priv->assigned_node);
g_list_free(rsc->priv->actions);
g_list_free(rsc->priv->active_nodes);
g_list_free(rsc->priv->launched);
g_list_free(rsc->priv->dangling_migration_sources);
g_list_free(rsc->priv->with_this_colocations);
g_list_free(rsc->priv->this_with_colocations);
g_list_free(rsc->priv->location_constraints);
g_list_free(rsc->priv->ticket_constraints);
if (rsc->priv->meta != NULL) {
g_hash_table_destroy(rsc->priv->meta);
}
if (rsc->priv->utilization != NULL) {
g_hash_table_destroy(rsc->priv->utilization);
}
if (rsc->priv->probed_nodes != NULL) {
g_hash_table_destroy(rsc->priv->probed_nodes);
}
if (rsc->priv->allowed_nodes != NULL) {
g_hash_table_destroy(rsc->priv->allowed_nodes);
}
free(rsc->priv);
free(rsc);
}
/*!
* \internal
* \brief Count a node and update most preferred to it as appropriate
*
* \param[in] rsc An active resource
* \param[in] node A node that \p rsc is active on
* \param[in,out] active This will be set to \p node if \p node is more
* preferred than the current value
* \param[in,out] count_all If not NULL, this will be incremented
* \param[in,out] count_clean If not NULL, this will be incremented if \p node
* is online and clean
*
* \return true if the count should continue, or false if sufficiently known
*/
bool
pe__count_active_node(const pcmk_resource_t *rsc, pcmk_node_t *node,
pcmk_node_t **active, unsigned int *count_all,
unsigned int *count_clean)
{
bool keep_looking = false;
bool is_happy = false;
CRM_CHECK((rsc != NULL) && (node != NULL) && (active != NULL),
return false);
is_happy = node->details->online && !node->details->unclean;
if (count_all != NULL) {
++*count_all;
}
if ((count_clean != NULL) && is_happy) {
++*count_clean;
}
if ((count_all != NULL) || (count_clean != NULL)) {
keep_looking = true; // We're counting, so go through entire list
}
if (rsc->priv->partial_migration_source != NULL) {
if (pcmk__same_node(node, rsc->priv->partial_migration_source)) {
*active = node; // This is the migration source
} else {
keep_looking = true;
}
} else if (!pcmk_is_set(rsc->flags, pcmk__rsc_needs_fencing)) {
if (is_happy && ((*active == NULL) || !(*active)->details->online
|| (*active)->details->unclean)) {
*active = node; // This is the first clean node
} else {
keep_looking = true;
}
}
if (*active == NULL) {
*active = node; // This is the first node checked
}
return keep_looking;
}
// Shared implementation of pcmk__rsc_methods_t:active_node()
static pcmk_node_t *
active_node(const pcmk_resource_t *rsc, unsigned int *count_all,
unsigned int *count_clean)
{
pcmk_node_t *active = NULL;
if (count_all != NULL) {
*count_all = 0;
}
if (count_clean != NULL) {
*count_clean = 0;
}
if (rsc == NULL) {
return NULL;
}
for (GList *iter = rsc->priv->active_nodes;
iter != NULL; iter = iter->next) {
if (!pe__count_active_node(rsc, (pcmk_node_t *) iter->data, &active,
count_all, count_clean)) {
break; // Don't waste time iterating if we don't have to
}
}
return active;
}
/*!
* \brief
* \internal Find and count active nodes according to \c PCMK_META_REQUIRES
*
* \param[in] rsc Resource to check
* \param[out] count If not NULL, will be set to count of active nodes
*
* \return An active node (or NULL if resource is not active anywhere)
*
* \note This is a convenience wrapper for active_node() where the count of all
* active nodes or only clean active nodes is desired according to the
* \c PCMK_META_REQUIRES meta-attribute.
*/
pcmk_node_t *
pe__find_active_requires(const pcmk_resource_t *rsc, unsigned int *count)
{
if (rsc == NULL) {
if (count != NULL) {
*count = 0;
}
return NULL;
}
if (pcmk_is_set(rsc->flags, pcmk__rsc_needs_fencing)) {
return rsc->priv->fns->active_node(rsc, count, NULL);
} else {
return rsc->priv->fns->active_node(rsc, NULL, count);
}
}
void
pe__count_common(pcmk_resource_t *rsc)
{
if (rsc->priv->children != NULL) {
for (GList *item = rsc->priv->children;
item != NULL; item = item->next) {
pcmk_resource_t *child = item->data;
child->priv->fns->count(item->data);
}
} else if (!pcmk_is_set(rsc->flags, pcmk__rsc_removed)
|| (rsc->priv->orig_role > pcmk_role_stopped)) {
rsc->priv->scheduler->priv->ninstances++;
if (pe__resource_is_disabled(rsc)) {
rsc->priv->scheduler->priv->disabled_resources++;
}
if (pcmk_is_set(rsc->flags, pcmk__rsc_blocked)) {
rsc->priv->scheduler->priv->blocked_resources++;
}
}
}
/*!
* \internal
* \brief Update a resource's next role
*
* \param[in,out] rsc Resource to be updated
* \param[in] role Resource's new next role
* \param[in] why Human-friendly reason why role is changing (for logs)
*/
void
pe__set_next_role(pcmk_resource_t *rsc, enum rsc_role_e role, const char *why)
{
pcmk__assert((rsc != NULL) && (why != NULL));
if (rsc->priv->next_role != role) {
pcmk__rsc_trace(rsc, "Resetting next role for %s from %s to %s (%s)",
rsc->id, pcmk_role_text(rsc->priv->next_role),
pcmk_role_text(role), why);
rsc->priv->next_role = role;
}
}
diff --git a/xml/nvset-4.0.rng b/xml/nvset-4.0.rng
index 0a12a10a4f..fb76e6d71c 100644
--- a/xml/nvset-4.0.rng
+++ b/xml/nvset-4.0.rng
@@ -1,61 +1,108 @@