shreyas-goenka · Oct 26, 2026 · Oct 26, 2026 · Oct 27, 2026 · Oct 27, 2026 · Oct 26, 2026
@@ -174,23 +174,26 @@ def get_last_version_id(target):
 
 
 def print_recorded_state(filename, target):
-    """Print the state file with its resources filled in from the deployment metadata service.
+    """Print DMS resources using the state file's header.
 
-    While recording, the file itself carries only the header - the service holds the resources - so
-    printing it raw would show an empty state and differ from the same test's non-recording run.
+    This checks history uploads even though resources.json remains authoritative.
+    Use --no-dms to inspect the file state instead.
     """
     data = json.loads(open(filename).read())
     # Recording stamps each resource payload with the deployment and version; drop it here so the
     # printed state matches a non-recording run without every caller piping through nostamp.
     data["state"] = scrub(get_recorded_state(target))
 
-    # The file's serial is a copy of the DMS version to ensure outdated resource state is not read.
-    # Rebuild in header order with the authoritative serial from the deployment.
+    # Pair DMS resources with the DMS version. The file serial can advance independently.
     last_version_id = get_last_version_id(target)
     serial = int(last_version_id) if last_version_id else 0
 
     # Recording itself is not what these tests assert, so drop the feature that marks it.
-    features = {k: v for k, v in (data.get("features") or {}).items() if k != "deployment_history"}
+    features = {
+        k: v
+        for k, v in (data.get("features") or {}).items()
+        if k not in {"deployment_history", "deployment_history_shadow"}
+    }
 
     rebuilt = {}
     for key in ("state_version", "cli_version", "lineage"):
@@ -213,7 +216,7 @@ def main():
         "--no-dms",
         action="store_true",
         help="Print resources.json as-is without fetching state from DMS. "
-        "Use in bundle/dms tests to assert the tombstone content of the file.",
+        "Use in bundle/dms tests to assert the file state and its feature markers.",
     )
     args = parser.parse_args()
 

@@ -5,6 +5,6 @@
   "serial": 0,
   "plan": {},
   "features": {
-    "deployment_history": {}
+    "deployment_history_shadow": {}
   }
 }
@@ -1,8 +1,5 @@
 bundle:
   name: dms-bind-unbind-$UNIQUE_NAME
-experimental:
-  deployment_history: true
-
 resources:
   jobs:
     foo:

@@ -4,13 +4,17 @@
 Error: bind is not supported for a bundle target that records deployment history
 
 
-=== Deploy so the bundle records state, then unbind is refused too
+=== Unbind is allowed for file-backed history state
 >>> [CLI] bundle deploy
 Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-bind-unbind-[UNIQUE_NAME]/default/files...
 Created jobs.foo
 Files: 2 uploaded, 0 deleted
 Resources: 1 created, 0 changed, 0 deleted, 0 unchanged
 
->>> musterr [CLI] bundle deployment unbind foo
-Error: unbind is not supported for a bundle target that records deployment history
+>>> [CLI] bundle deployment unbind foo
 
+>>> jq {features, state} .databricks/bundle/default/resources.json
+{
+  "features": null,
+  "state": {}
+}
@@ -8,7 +8,8 @@ job_id=$($CLI jobs create --json '{"name": "bind-target-'$UNIQUE_NAME'"}' | jq -
 add_repl "$job_id" JOB_ID
 trace musterr $CLI bundle deployment bind foo "$job_id"
 
-# unbind refuses when it opens an existing recording state, so it needs a deployment first.
-title "Deploy so the bundle records state, then unbind is refused too"
+# File state remains writable even when it carries the history marker.
+title "Unbind is allowed for file-backed history state"
 trace $CLI bundle deploy
-trace musterr $CLI bundle deployment unbind foo
+trace $CLI bundle deployment unbind foo
+trace jq '{features, state}' .databricks/bundle/default/resources.json
@@ -67,7 +67,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged
   "lineage": "[UUID]",
   "serial": 1,
   "features": {
-    "deployment_history": {}
+    "deployment_history_shadow": {}
   }
 }
 
@@ -79,18 +79,10 @@ Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged
   "method": "GET",
   "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]"
 }
-{
-  "method": "GET",
-  "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/resources"
-}
 {
   "method": "GET",
   "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]"
 }
-{
-  "method": "GET",
-  "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/resources"
-}
 
 >>> print_state.py --no-dms
 {
@@ -99,9 +91,17 @@ Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged
  "lineage": "[UUID]",
  "serial": 1,
  "features": {
-  "deployment_history": {}
+  "deployment_history_shadow": {}
  },
- "state": {}
+ "state": {
+  "resources.schemas.foo": {
+   "__id__": "main.dms_declined_deploy_schema_[UNIQUE_NAME]",
+   "state": {
+    "catalog_name": "main",
+    "name": "dms_declined_deploy_schema_[UNIQUE_NAME]"
+   }
+  }
+ }
 }
 
 === A destructive change without --auto-approve is declined: this console cannot prompt
@@ -124,10 +124,6 @@ Files: 2 uploaded, 0 deleted
   "method": "GET",
   "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]"
 }
-{
-  "method": "GET",
-  "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/resources"
-}
 
 === The recorded resources are unchanged by the decline: the schema is still at version 1
 >>> MSYS_NO_PATHCONV=1 [CLI] bundle-deployments list-resources deployments/[DEPLOYMENT_ID] -o json
@@ -163,10 +159,6 @@ false
   "method": "GET",
   "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]"
 }
-{
-  "method": "GET",
-  "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/resources"
-}
 {
   "method": "POST",
   "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/versions",

@@ -5,7 +5,7 @@
   "plan_version": [PLAN_VERSION],
   "cli_version": "[CLI_VERSION]",
   "features": {
-    "deployment_history": {}
+    "deployment_history_shadow": {}
   }
 }
 
@@ -61,7 +61,7 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged
   "lineage": "[UUID]",
   "serial": 1,
   "features": {
-    "deployment_history": {}
+    "deployment_history_shadow": {}
   }
 }
 
@@ -91,9 +91,53 @@ Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged
  "lineage": "[UUID]",
  "serial": 1,
  "features": {
-  "deployment_history": {}
+  "deployment_history_shadow": {}
  },
- "state": {}
+ "state": {
+  "resources.jobs.child": {
+   "__id__": "[CHILD_ID]",
+   "state": {
+    "deployment": {
+     "deployment_id": "[DEPLOYMENT_ID]",
+     "kind": "BUNDLE",
+     "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-depends-on-[UNIQUE_NAME]/default/state/metadata.json",
+     "version_id": "1"
+    },
+    "description": "depends on [PARENT_ID]",
+    "edit_mode": "UI_LOCKED",
+    "format": "MULTI_TASK",
+    "max_concurrent_runs": 1,
+    "name": "child",
+    "queue": {
+     "enabled": true
+    }
+   },
+   "depends_on": [
+    {
+     "node": "resources.jobs.parent",
+     "label": "${resources.jobs.parent.id}"
+    }
+   ]
+  },
+  "resources.jobs.parent": {
+   "__id__": "[PARENT_ID]",
+   "state": {
+    "deployment": {
+     "deployment_id": "[DEPLOYMENT_ID]",
+     "kind": "BUNDLE",
+     "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-depends-on-[UNIQUE_NAME]/default/state/metadata.json",
+     "version_id": "1"
+    },
+    "edit_mode": "UI_LOCKED",
+    "format": "MULTI_TASK",
+    "max_concurrent_runs": 1,
+    "name": "parent",
+    "queue": {
+     "enabled": true
+    }
+   }
+  }
+ }
 }
 
 >>> [CLI] bundle destroy --auto-approve

@@ -67,7 +67,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged
   "lineage": "[UUID]",
   "serial": 1,
   "features": {
-    "deployment_history": {}
+    "deployment_history_shadow": {}
   }
 }
 
@@ -128,7 +128,7 @@ A common practice is to use a username or principal name in this path, i.e. use
   "lineage": "[UUID]",
   "serial": 1,
   "features": {
-    "deployment_history": {}
+    "deployment_history_shadow": {}
   }
 }
 

@@ -138,7 +138,7 @@ Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged
   "lineage": "[UUID]",
   "serial": 2,
   "features": {
-    "deployment_history": {}
+    "deployment_history_shadow": {}
   }
 }
 
@@ -163,9 +163,17 @@ Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged
  "lineage": "[UUID]",
  "serial": 2,
  "features": {
-  "deployment_history": {}
+  "deployment_history_shadow": {}
  },
- "state": {}
+ "state": {
+  "resources.schemas.foo": {
+   "__id__": "main.dms_empty_grants_[UNIQUE_NAME]",
+   "state": {
+    "catalog_name": "main",
+    "name": "dms_empty_grants_[UNIQUE_NAME]"
+   }
+  }
+ }
 }
 
 >>> [CLI] bundle destroy --auto-approve

@@ -25,29 +25,37 @@ Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged
   "features": null
 }
 
-=== Enabling recording is refused because the existing deployment is not recorded
+=== The environment flag enables best-effort history for the existing deployment
 >>> DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=true [CLI] bundle deploy
-Error: deployment history setting (true) does not match the existing state (false)
-
-Update experimental.deployment_history to match the existing deployment, or run "databricks bundle destroy" to start over
-
+Enabling deployment history...
+Deployment history enabled: [DATABRICKS_URL]/deployments/[DEPLOYMENT_ID]?version=2
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-existing-state-[UNIQUE_NAME]/default/files...
+Updated jobs.one
+Files: 0 uploaded, 0 deleted
+Resources: 0 created, 1 changed, 0 deleted, 0 unchanged
 
 >>> print_requests.py --dms //api/2.0/bundle --oneline
-
-=== Wiping the local cache still refuses after pulling the non-DMS state from the workspace
+{"method": "POST", "path": "/api/2.0/bundle/deployments", "body": {"display_name": "dms-existing-state-[UNIQUE_NAME]", "initial_parent_path": "/Workspace/Users/[USERNAME]/.bundle/dms-existing-state-[UNIQUE_NAME]/default/state", "target_name": "default", "workspace_info": {"file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-existing-state-[UNIQUE_NAME]/default/files", "root_path": "/Workspace/Users/[USERNAME]/.bundle/dms-existing-state-[UNIQUE_NAME]/default"}}}
+{"method": "POST", "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/versions", "q": {"version_id": "2"}, "body": {"cli_version": "[CLI_VERSION]", "operations": [{"action_type": "OPERATION_ACTION_TYPE_INITIAL_REGISTER", "resource_key": "jobs.one"}], "version_type": "VERSION_TYPE_DEPLOY"}}
+{"method": "PATCH", "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/versions/2/operations/jobs.one", "q": {"update_mask": "state,error_message,resource_id,status"}, "body": {"error_message": "", "resource_id": "[ONE_ID]", "sequence_id": 0, "state": "{\"state\":{\"deployment\":{\"kind\":\"BUNDLE\",\"metadata_file_path\":\"/Workspace/Users/[USERNAME]/.bundle/dms-existing-state-[UNIQUE_NAME]/default/state/metadata.json\"},\"edit_mode\":\"UI_LOCKED\",\"format\":\"MULTI_TASK\",\"max_concurrent_runs\":1,\"name\":\"one\",\"queue\":{\"enabled\":true}}}", "status": "OPERATION_STATUS_SUCCEEDED"}}
+{"method": "POST", "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/versions/2/complete", "body": {"completion_reason": "VERSION_COMPLETE_SUCCESS"}}
+{"method": "POST", "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/versions", "q": {"version_id": "3"}, "body": {"cli_version": "[CLI_VERSION]", "operations": [{"action_type": "OPERATION_ACTION_TYPE_UPDATE", "resource_key": "jobs.one"}], "previous_version_id": "2", "version_type": "VERSION_TYPE_DEPLOY"}}
+{"method": "PATCH", "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/versions/3/operations/jobs.one", "q": {"update_mask": "state,error_message,resource_id,status"}, "body": {"error_message": "", "resource_id": "[ONE_ID]", "sequence_id": 0, "state": "{\"state\":{\"deployment\":{\"deployment_id\":\"[DEPLOYMENT_ID]\",\"kind\":\"BUNDLE\",\"metadata_file_path\":\"/Workspace/Users/[USERNAME]/.bundle/dms-existing-state-[UNIQUE_NAME]/default/state/metadata.json\",\"version_id\":\"3\"},\"edit_mode\":\"UI_LOCKED\",\"format\":\"MULTI_TASK\",\"max_concurrent_runs\":1,\"name\":\"one\",\"queue\":{\"enabled\":true}}}", "status": "OPERATION_STATUS_SUCCEEDED"}}
+{"method": "POST", "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/versions/3/complete", "body": {"completion_reason": "VERSION_COMPLETE_SUCCESS"}}
+
+=== Wiping the local cache pulls history-enabled file state without repeating registration
 >>> DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=true [CLI] bundle deploy
-Error: deployment history setting (true) does not match the existing state (false)
-
-Update experimental.deployment_history to match the existing deployment, or run "databricks bundle destroy" to start over
-
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-existing-state-[UNIQUE_NAME]/default/files...
+Files: 2 uploaded, 0 deleted
+Resources: 0 created, 0 changed, 0 deleted, 1 unchanged
 
 === A leftover local WAL with the wrong lineage is still refused
 >>> DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=true [CLI] bundle deploy
 Error: reading state from [TEST_TMP_DIR]/.databricks/bundle/default/resources.json: WAL recovery failed: WAL lineage ("stale") does not match state lineage ("[UUID]")
 
 
 === Destroy clears the deployment. With nothing recorded remotely the bundle is treated as new, so recording can be enabled once the stale WAL is gone
->>> DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=true [CLI] bundle destroy --auto-approve
+>>> DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=false [CLI] bundle destroy --auto-approve
 The following resources will be deleted:
   delete resources.jobs.one
 
@@ -62,17 +70,21 @@ Files: 2 uploaded, 0 deleted
 Resources: 1 created, 0 changed, 0 deleted, 0 unchanged
 
 >>> print_requests.py --dms //api/2.0/bundle --oneline
+{"method": "POST", "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/versions", "q": {"version_id": "4"}, "body": {"cli_version": "[CLI_VERSION]", "operations": [{"action_type": "OPERATION_ACTION_TYPE_DELETE", "resource_key": "jobs.one"}], "previous_version_id": "3", "version_type": "VERSION_TYPE_DESTROY"}}
+{"method": "PATCH", "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/versions/4/operations/jobs.one", "q": {"update_mask": "state,error_message,status"}, "body": {"error_message": "", "sequence_id": 0, "status": "OPERATION_STATUS_SUCCEEDED"}}
+{"method": "POST", "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/versions/4/complete", "body": {"completion_reason": "VERSION_COMPLETE_SUCCESS"}}
+{"method": "DELETE", "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]"}
 {"method": "POST", "path": "/api/2.0/bundle/deployments", "body": {"display_name": "dms-existing-state-[UNIQUE_NAME]", "initial_parent_path": "/Workspace/Users/[USERNAME]/.bundle/dms-existing-state-[UNIQUE_NAME]/default/state", "target_name": "default", "workspace_info": {"file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-existing-state-[UNIQUE_NAME]/default/files", "root_path": "/Workspace/Users/[USERNAME]/.bundle/dms-existing-state-[UNIQUE_NAME]/default"}}}
-{"method": "POST", "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/versions", "q": {"version_id": "1"}, "body": {"cli_version": "[CLI_VERSION]", "operations": [{"action_type": "OPERATION_ACTION_TYPE_CREATE", "resource_key": "jobs.one"}], "version_type": "VERSION_TYPE_DEPLOY"}}
-{"method": "PATCH", "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/versions/1/operations/jobs.one", "q": {"update_mask": "state,error_message,resource_id,status"}, "body": {"error_message": "", "resource_id": "[ONE_ID]", "sequence_id": 0, "state": "{\"state\":{\"deployment\":{\"deployment_id\":\"[DEPLOYMENT_ID]\",\"kind\":\"BUNDLE\",\"metadata_file_path\":\"/Workspace/Users/[USERNAME]/.bundle/dms-existing-state-[UNIQUE_NAME]/default/state/metadata.json\",\"version_id\":\"1\"},\"edit_mode\":\"UI_LOCKED\",\"format\":\"MULTI_TASK\",\"max_concurrent_runs\":1,\"name\":\"one\",\"queue\":{\"enabled\":true}}}", "status": "OPERATION_STATUS_SUCCEEDED"}}
-{"method": "POST", "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/versions/1/complete", "body": {"completion_reason": "VERSION_COMPLETE_SUCCESS"}}
+{"method": "POST", "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID_2]/versions", "q": {"version_id": "1"}, "body": {"cli_version": "[CLI_VERSION]", "operations": [{"action_type": "OPERATION_ACTION_TYPE_CREATE", "resource_key": "jobs.one"}], "version_type": "VERSION_TYPE_DEPLOY"}}
+{"method": "PATCH", "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID_2]/versions/1/operations/jobs.one", "q": {"update_mask": "state,error_message,resource_id,status"}, "body": {"error_message": "", "resource_id": "[ONE_ID_2]", "sequence_id": 0, "state": "{\"state\":{\"deployment\":{\"deployment_id\":\"[DEPLOYMENT_ID_2]\",\"kind\":\"BUNDLE\",\"metadata_file_path\":\"/Workspace/Users/[USERNAME]/.bundle/dms-existing-state-[UNIQUE_NAME]/default/state/metadata.json\",\"version_id\":\"1\"},\"edit_mode\":\"UI_LOCKED\",\"format\":\"MULTI_TASK\",\"max_concurrent_runs\":1,\"name\":\"one\",\"queue\":{\"enabled\":true}}}", "status": "OPERATION_STATUS_SUCCEEDED"}}
+{"method": "POST", "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID_2]/versions/1/complete", "body": {"completion_reason": "VERSION_COMPLETE_SUCCESS"}}
 
 >>> find.py resources[.]json[.]wal --expect 0
 
 >>> jq {state_version, features} .databricks/bundle/default/resources.json
 {
   "state_version": 3,
   "features": {
-    "deployment_history": {}
+    "deployment_history_shadow": {}
   }
 }
@@ -2,29 +2,32 @@ envsubst < databricks.yml.tmpl > databricks.yml
 
 title "Deploy without recording: ordinary direct-engine state tracking one job, unknown to DMS"
 trace $CLI bundle deploy
+replace_ids.py
 dms_plan
 trace print_requests.py --dms //api/2.0/bundle --oneline
 trace jq '{state_version, features}' .databricks/bundle/default/resources.json
 
-title "Enabling recording is refused because the existing deployment is not recorded"
-musterr trace DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=true $CLI bundle deploy
+title "The environment flag enables best-effort history for the existing deployment"
+trace DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=true $CLI bundle deploy
+deployment_id=$(MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-existing-state-${UNIQUE_NAME}/default/state/resources.deployment.json" | python3 -c 'import sys, json; print(json.load(sys.stdin)["object_id"])')
+add_repl "$deployment_id" DEPLOYMENT_ID
 trace print_requests.py --dms //api/2.0/bundle --oneline
 
-title "Wiping the local cache still refuses after pulling the non-DMS state from the workspace"
+title "Wiping the local cache pulls history-enabled file state without repeating registration"
 rm -rf .databricks
-musterr trace DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=true $CLI bundle deploy
+trace DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=true $CLI bundle deploy
 
 title "A leftover local WAL with the wrong lineage is still refused"
 mkdir -p .databricks/bundle/default
-printf '{"lineage":"stale","serial":99,"state_version":2,"features":{"deployment_history":{}}}\n' > .databricks/bundle/default/resources.json.wal
+printf '{"lineage":"stale","serial":99,"state_version":2,"features":{"deployment_history_shadow":{}}}\n' > .databricks/bundle/default/resources.json.wal
 musterr trace DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=true $CLI bundle deploy
 
 title "Destroy clears the deployment. With nothing recorded remotely the bundle is treated as new, so recording can be enabled once the stale WAL is gone"
-# Clear the stale WAL first: it was refused, not deleted, and destroy runs without recording, so it
+# Clear the stale WAL first: it was refused, not deleted, so destroy
 # would otherwise try to replay it and fail on the lineage mismatch.
 rm -f .databricks/bundle/default/resources.json.wal
 # Keep the mismatched setting here to verify that destroy ignores configuration and follows state.
-trace DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=true $CLI bundle destroy --auto-approve
+trace DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=false $CLI bundle destroy --auto-approve
 trace DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=true $CLI bundle deploy
 trace print_requests.py --dms //api/2.0/bundle --oneline
 trace find.py 'resources[.]json[.]wal' --expect 0