acc: replace LocalOnly option with Local & Cloud (#2387)

## Changes
Instead of LocalOnly with non-composable semantics there are two
composable options:
- Local - enable test locally
- Cloud - enable test on the cloud

By default Cloud is switched off except in bundle (but not in
bundle/variables and bundle/help).

## Tests
Using this in #2383 to have test that runs on cloud but not locally.

Author: denik

acceptance/acceptance_test.go

@@ -217,8 +217,12 @@ func runTest(t *testing.T, dir, coverDir string, repls testdiff.ReplacementsCont
 	}
 
 	cloudEnv := os.Getenv("CLOUD_ENV")
-	if isTruePtr(config.LocalOnly) && cloudEnv != "" {
-		t.Skipf("Disabled via LocalOnly setting in %s (CLOUD_ENV=%s)", configPath, cloudEnv)
+	if !isTruePtr(config.Local) && cloudEnv == "" {
+		t.Skipf("Disabled via Local setting in %s (CLOUD_ENV=%s)", configPath, cloudEnv)
+	}
+
+	if !isTruePtr(config.Cloud) && cloudEnv != "" {
+		t.Skipf("Disabled via Cloud setting in %s (CLOUD_ENV=%s)", configPath, cloudEnv)
 	}
 
 	var tmpDir string

acceptance/auth/bundle_and_profile/test.toml

@@ -1,5 +1,3 @@
-LocalOnly=true
-
 # Some of the clouds have DATABRICKS_HOST variable setup without https:// prefix
 # In the result, output is replaced with DATABRICKS_URL variable instead of DATABRICKS_HOST
 # This is a workaround to replace DATABRICKS_URL with DATABRICKS_HOST

acceptance/auth/credentials/test.toml

@@ -1,5 +1,3 @@
-LocalOnly = true
-
 RecordRequests = true
 IncludeRequestHeaders = ["Authorization", "User-Agent"]
 

acceptance/bundle/debug/test.toml

@@ -1,4 +1,4 @@
-LocalOnly = true
+Cloud = false
 
 [[Repls]]
 # The keys are unsorted and also vary per OS

acceptance/bundle/generate/git_job/test.toml

@@ -1,4 +1,4 @@
-LocalOnly = true  # This test needs to run against stubbed Databricks API
+Cloud = false  # This test needs to run against stubbed Databricks API
 
 [[Server]]
 Pattern = "GET /api/2.1/jobs/get"

acceptance/bundle/help/test.toml

@@ -0,0 +1 @@
+Cloud = false

acceptance/bundle/libraries/maven/test.toml

@@ -1,5 +1,5 @@
 # We run this test only locally for now because we need to figure out how to do
 # bundle destroy on script.cleanup first.
-LocalOnly = true
+Cloud = false
 
 RecordRequests = true

acceptance/bundle/libraries/pypi/test.toml

@@ -1,5 +1,5 @@
 # We run this test only locally for now because we need to figure out how to do
 # bundle destroy on script.cleanup first.
-LocalOnly = true
+Cloud = false
 
 RecordRequests = true

acceptance/bundle/templates-machinery/helpers-error/test.toml

@@ -1,5 +1,4 @@
 Badness = '''(minor) error message is not great: executing "" at <user_name>: error calling user_name:'''
-LocalOnly = true
 
 [[Server]]
 Pattern = "GET /api/2.0/preview/scim/v2/Me"

acceptance/bundle/templates-machinery/helpers/test.toml

@@ -1,2 +1 @@
-# Testing template machinery, by default there is no need to check against cloud.
-LocalOnly = true
+Cloud = false

acceptance/bundle/templates-machinery/test.toml

@@ -1,2 +1,2 @@
 # At the moment, there are many differences across different envs w.r.t to catalog use, node type and so on.
-LocalOnly = true
+Cloud = false

acceptance/bundle/templates/test.toml

@@ -0,0 +1,2 @@
+Local = true
+Cloud = true

acceptance/bundle/test.toml

@@ -1,6 +1,6 @@
 # Since we use existing cluster id value which is not available in cloud envs, we need to stub the request
 # and run this test only locally
-LocalOnly = true
+Cloud = false
 
 [[Server]]
 Pattern = "GET /api/2.1/clusters/get"

acceptance/bundle/trampoline/warning_message_with_new_spark/test.toml

@@ -1,6 +1,6 @@
 # Since we use existing cluster id value which is not available in cloud envs, we need to stub the request
 # and run this test only locally
-LocalOnly = true
+Cloud = false
 
 [[Server]]
 Pattern = "GET /api/2.1/clusters/get"

acceptance/bundle/trampoline/warning_message_with_old_spark/test.toml

@@ -1,3 +1,3 @@
 # The tests here intend to test variable interpolation via "bundle validate".
 # Even though "bundle validate" does a few API calls, that's not the focus there.
-LocalOnly = true
+Cloud = false

acceptance/bundle/variables/test.toml

@@ -1,4 +1,3 @@
-LocalOnly = true
 RecordRequests = true
 
 [[Server]]

acceptance/cmd/workspace/apps/test.toml

@@ -24,8 +24,11 @@ type TestConfig struct {
 	// If absent, default to true.
 	GOOS map[string]bool
 
-	// If true, do not run this test against cloud environment
-	LocalOnly *bool
+	// If true, run this test when running locally with a testserver
+	Local *bool
+
+	// If true, run this test when running with cloud env configured
+	Cloud *bool
 
 	// List of additional replacements to apply on this test.
 	// Old is a regexp, New is a replacement expression.

acceptance/config_test.go

@@ -1,4 +1,3 @@
-LocalOnly = true
 RecordRequests = true
 
 [[Server]]

acceptance/selftest/server/test.toml

@@ -1,3 +1,6 @@
+Local = true
+Cloud = true
+
 [[Repls]]
 Old = 'Read complete after [^\s]+'
 New = 'Read complete after (redacted)'

acceptance/selftest/test.toml

@@ -0,0 +1,3 @@
+# Default settings that apply to all tests unless overriden by test.toml files in inner directories.
+Local = true
+Cloud = false

acceptance/terraform/test.toml

@@ -1,4 +1,3 @@
-LocalOnly = true  # request recording currently does not work with cloud environment
 RecordRequests = true
 
 [[Server]]

acceptance/test.toml

@@ -1,4 +1,3 @@
-LocalOnly = true  # request recording currently does not work with cloud environment
 RecordRequests = true
 IncludeRequestHeaders = ["Authorization", "User-Agent"]