Refactor eco testing

.github/workflows/tox.yml

@@ -41,7 +41,6 @@ jobs:
             hook
             docs
             schemas
-            eco
             pre
             py311-devel
             py310-lower
@@ -110,6 +109,7 @@ jobs:
         with:
           path: |
             .cache/eco
+            test/fixtures/eco/*/.tox
             examples/playbooks/collections/ansible_collections
             ~/.cache/ansible-compat
             ~/.ansible/collections

.gitignore

@@ -82,3 +82,4 @@ node_modules
 .direnv
 uv.lock
 .ansible
+test/fixtures/eco

.gitmodules

@@ -0,0 +1,21 @@
+[submodule "test/fixtures/eco/bootstrap"]
+	path = test/fixtures/eco/bootstrap
+	url = https://github.com/robertdebock/ansible-role-bootstrap
+[submodule "test/fixtures/eco/cisco.nxos"]
+	path = test/fixtures/eco/cisco.nxos
+	url = https://github.com/ansible-collections/cisco.nxos
+[submodule "test/fixtures/eco/ansible_collection_system"]
+	path = test/fixtures/eco/ansible_collection_system
+	url = https://github.com/devroles/ansible_collection_system
+[submodule "test/fixtures/eco/ansible-docker-rootless"]
+	path = test/fixtures/eco/ansible-docker-rootless
+	url = https://github.com/konstruktoid/ansible-docker-rootless
+[submodule "test/fixtures/eco/ansible-role-hardening"]
+	path = test/fixtures/eco/ansible-role-hardening
+	url = https://github.com/konstruktoid/ansible-role-hardening
+[submodule "test/fixtures/eco/ansible-role-mysql"]
+	path = test/fixtures/eco/ansible-role-mysql
+	url = https://github.com/geerlingguy/ansible-role-mysql.git
+[submodule "test/fixtures/eco/debops"]
+	path = test/fixtures/eco/debops
+	url = https://github.com/debops/debops

.pre-commit-config.yaml

@@ -22,6 +22,7 @@ exclude: >
     examples/playbooks/vars/empty.transformed.yml|
     examples/playbooks/vars/empty.yml|
     src/ansiblelint/schemas/rulebook.json|
+    test/fixtures/eco/.*|
     test/schemas/data/licenses.json|
     test/schemas/negative_test|
     test/schemas/package-lock.json

codecov.yml

@@ -2,7 +2,6 @@
 comment: false
 coverage:
   status:
-    patch: true
+    patch: false # avoid false positives
     project:
-      default:
-        threshold: 0.5%
+      default: true

cspell.config.yaml

@@ -19,6 +19,7 @@ ignorePaths:
   - docs/requirements.in
   # Test fixtures generated from outside
   - test/**/*.result
+  - test/fixtures/eco/**
   - src/ansiblelint/schemas/*.json
   # Other
   - "*.svg"

pyproject.toml

@@ -51,7 +51,7 @@ repository = "https://github.com/ansible/ansible-lint"
 [tool.codespell]
 # indention is a typo in ruamel.yaml's API
 ignore-words-list = "indention"
-skip = ".tox,.mypy_cache,build,.git,.eggs,pip-wheel-metadata"
+skip = ".tox,.mypy_cache,build,.git,.eggs,pip-wheel-metadata,test/fixtures/eco/*.json"
 
 # Keep this default because xml/report do not know to use load it from config file:
 # data_file = ".coverage"
@@ -86,7 +86,7 @@ disallow_untyped_calls = true
 disallow_untyped_defs = true
 error_summary = true
 # site-packages is here to help vscode mypy integration getting confused
-exclude = "(.cache|.config|.eggs|.tox|_readthedocs|build|dist|test/local-content|site|site-packages|~/.pyenv|examples|plugins/modules).*"
+exclude = "(.cache|.config|.eggs|.tox|_readthedocs|build|dist|test/local-content|site|site-packages|~/.pyenv|examples|plugins/modules|test/fixtures/eco).*"
 # https://github.com/python/mypy/issues/12664
 follow_untyped_imports = true
 incremental = false
@@ -392,7 +392,7 @@ reportPrivateImportUsage = false
 [tool.pytest.ini_options]
 # do not add options here as this will likely break either console runs or IDE
 # integration like vscode or pycharm
-addopts = "-p no:pytest_cov --durations=10 --failed-first"
+addopts = "-p no:pytest_cov --durations=10 --failed-first -m 'not eco'"
 doctest_optionflags = ["ALLOW_UNICODE", "ELLIPSIS"]
 filterwarnings = [
   "error",
@@ -409,6 +409,9 @@ filterwarnings = [
   "ignore:Attribute s is deprecated and will be removed in Python 3.14; use value instead:DeprecationWarning"
 ]
 junit_family = "legacy"
+markers = [
+  "eco: test impact on ecosystem repositories (slow)"
+]
 minversion = "4.6.6"
 # https://code.visualstudio.com/docs/python/testing
 # coverage is re-enabled in `tox.ini`. That approach is safer than
@@ -435,6 +438,7 @@ norecursedirs = [
   "plugins",
   "site",
   "src/ansible_lint.egg-info",
+  "test/fixtures/eco",
   "test/schemas"
 ]
 python_files = [
@@ -574,6 +578,7 @@ exclude = [
   "collections",
   "examples",
   "plugins",
+  "test/fixtures/eco",
   "test/local-content",
   "venv",
   "src/ansiblelint/_version.py"

test/fixtures/eco/ansible-role-hardening.ignore.txt

@@ -0,0 +1,2 @@
+# This file contains ignores rule violations for ansible-lint
+molecule/default/verify.yml name[unique]

test/fixtures/eco/ansible-role-hardening.json

@@ -0,0 +1,3 @@
+name[unique]: Task name 'Get PAM version' is not unique. It was first used on line 622.
+molecule/default/verify.yml:1124
+

test/fixtures/eco/ansible-role-mysql.ignore.txt

@@ -0,0 +1,12 @@
+# This file contains ignores rule violations for ansible-lint
+handlers/main.yml name[casing]
+meta/main.yml schema[meta]
+tasks/databases.yml fqcn[action]
+tasks/main.yml name[missing]
+tasks/replication.yml fqcn[action]
+tasks/replication.yml ignore-errors
+tasks/secure-installation.yml fqcn[action]
+tasks/secure-installation.yml jinja[spacing]
+tasks/secure-installation.yml no-changed-when
+tasks/setup-RedHat.yml fqcn[action-core]
+tasks/users.yml fqcn[action]

test/fixtures/eco/ansible-role-mysql.json

@@ -0,0 +1,100 @@
+name[casing]: All names should start with an uppercase letter.
+handlers/main.yml:2:9 Task/Handler: restart mysql
+
+schema[meta]: $.galaxy_info.min_ansible_version 2.12 is not of type 'string'. See https://docs.ansible.com/ansible/latest/playbook_guide/playbooks_reuse_roles.html#using-role-dependencies
+meta/main.yml:1  Returned errors will not include exact line numbers, but they will mention
+the schema name being used as a tag, like ``schema[playbook]``,
+``schema[tasks]``.
+
+This rule is not skippable and stops further processing of the file.
+
+If incorrect schema was picked, you might want to either:
+
+* move the file to standard location, so its file is detected correctly.
+* use ``kinds:`` option in linter config to help it pick correct file type.
+[/]
+
+fqcn[action]: Use FQCN for module actions, such `community.mysql.mysql_db`.
+tasks/databases.yml:3:3 Action `mysql_db` is not FQCN.
+
+name[missing]: All tasks should be named.
+tasks/main.yml:3 Task/Handler: include_tasks variables.yml
+
+name[missing]: All tasks should be named.
+tasks/main.yml:6 Task/Handler: include_tasks setup-RedHat.yml
+
+name[missing]: All tasks should be named.
+tasks/main.yml:9 Task/Handler: include_tasks setup-Debian.yml
+
+name[missing]: All tasks should be named.
+tasks/main.yml:12 Task/Handler: include_tasks setup-Archlinux.yml
+
+name[missing]: All tasks should be named.
+tasks/main.yml:22 Task/Handler: include_tasks configure.yml
+
+name[missing]: All tasks should be named.
+tasks/main.yml:23 Task/Handler: include_tasks secure-installation.yml
+
+name[missing]: All tasks should be named.
+tasks/main.yml:24 Task/Handler: include_tasks databases.yml
+
+name[missing]: All tasks should be named.
+tasks/main.yml:25 Task/Handler: include_tasks users.yml
+
+name[missing]: All tasks should be named.
+tasks/main.yml:26 Task/Handler: include_tasks replication.yml
+
+fqcn[action]: Use FQCN for module actions, such `community.mysql.mysql_user`.
+tasks/replication.yml:3:3 Action `mysql_user` is not FQCN.
+
+fqcn[action]: Use FQCN for module actions, such `community.mysql.mysql_replication`.
+tasks/replication.yml:18:3 Action `mysql_replication` is not FQCN.
+
+fqcn[action]: Use FQCN for module actions, such `community.mysql.mysql_replication`.
+tasks/replication.yml:32:3 Action `mysql_replication` is not FQCN.
+
+ignore-errors: Use failed_when and specify error conditions instead of using ignore_errors.
+tasks/replication.yml:45 Task/Handler: Configure replication on the slave.
+
+fqcn[action]: Use FQCN for module actions, such `community.mysql.mysql_replication`.
+tasks/replication.yml:46:3 Action `mysql_replication` is not FQCN.
+
+fqcn[action]: Use FQCN for module actions, such `community.mysql.mysql_replication`.
+tasks/replication.yml:65:3 Action `mysql_replication` is not FQCN.
+
+fqcn[action]: Use FQCN for module actions, such `community.mysql.mysql_user`.
+tasks/secure-installation.yml:3:3 Action `mysql_user` is not FQCN.
+
+jinja[spacing]: Jinja2 spacing could be improved: {{ mysql_root_hosts.stdout_lines|default([]) }} -> {{ mysql_root_hosts.stdout_lines | default([]) }} (warning)
+tasks/secure-installation.yml:45 Jinja2 template rewrite recommendation: `{{ mysql_root_hosts.stdout_lines | default([]) }}`.
+
+no-changed-when: Commands should not change things if nothing needs doing.
+tasks/secure-installation.yml:45 Task/Handler: Update MySQL root authentication via socket for localhost (Linux, MySQL ≥ 8.4)
+
+jinja[spacing]: Jinja2 spacing could be improved: {{ mysql_root_hosts.stdout_lines|default([]) }} -> {{ mysql_root_hosts.stdout_lines | default([]) }} (warning)
+tasks/secure-installation.yml:61 Jinja2 template rewrite recommendation: `{{ mysql_root_hosts.stdout_lines | default([]) }}`.
+
+no-changed-when: Commands should not change things if nothing needs doing.
+tasks/secure-installation.yml:61 Task/Handler: Update MySQL root password for localhost root account (5.7.x ≤ MySQL < 8.4)
+
+jinja[spacing]: Jinja2 spacing could be improved: {{ mysql_root_hosts.stdout_lines|default([]) }} -> {{ mysql_root_hosts.stdout_lines | default([]) }} (warning)
+tasks/secure-installation.yml:80 Jinja2 template rewrite recommendation: `{{ mysql_root_hosts.stdout_lines | default([]) }}`.
+
+no-changed-when: Commands should not change things if nothing needs doing.
+tasks/secure-installation.yml:80 Task/Handler: Update MySQL root password for localhost root account (< 5.7.x).
+
+jinja[spacing]: Jinja2 spacing could be improved: {{ mysql_anonymous_hosts.stdout_lines|default([]) }} -> {{ mysql_anonymous_hosts.stdout_lines | default([]) }} (warning)
+tasks/secure-installation.yml:110 Jinja2 template rewrite recommendation: `{{ mysql_anonymous_hosts.stdout_lines | default([]) }}`.
+
+fqcn[action]: Use FQCN for module actions, such `community.mysql.mysql_user`.
+tasks/secure-installation.yml:111:3 Action `mysql_user` is not FQCN.
+
+fqcn[action]: Use FQCN for module actions, such `community.mysql.mysql_db`.
+tasks/secure-installation.yml:118:3 Action `mysql_db` is not FQCN.
+
+fqcn[action-core]: Use FQCN for builtin module actions (ansible.builtin.yum).
+tasks/setup-RedHat.yml:3:3 Use `ansible.builtin.dnf` or `ansible.legacy.dnf` instead.
+
+fqcn[action]: Use FQCN for module actions, such `community.mysql.mysql_user`.
+tasks/users.yml:3:3 Action `mysql_user` is not FQCN.
+

test/fixtures/eco/ansible_collection_system.ignore.txt

@@ -0,0 +1,81 @@
+# This file contains ignores rule violations for ansible-lint
+.ansible/collections/ansible_collections/devroles/system/roles/authorized_key/defaults/main.yml var-naming[no-role-prefix]
+.ansible/collections/ansible_collections/devroles/system/roles/dotfiles/tasks/03_vim.yml yaml[octal-values]
+.ansible/collections/ansible_collections/devroles/system/roles/firewalld/defaults/main.yml yaml[comments]
+.ansible/collections/ansible_collections/devroles/system/roles/mariadb/tasks/configure_access.yml yaml[octal-values]
+.ansible/collections/ansible_collections/devroles/system/roles/mounts/defaults/main.yml var-naming[no-role-prefix]
+.ansible/collections/ansible_collections/devroles/system/roles/mythtv/handlers/main.yml no-changed-when
+.ansible/collections/ansible_collections/devroles/system/roles/mythtv/tasks/main.yml yaml[octal-values]
+.ansible/collections/ansible_collections/devroles/system/roles/sudoers/defaults/main.yml var-naming[no-role-prefix]
+.ansible/collections/ansible_collections/devroles/system/roles/sudoers/tasks/main.yml yaml[octal-values]
+.ansible/collections/ansible_collections/devroles/system/roles/update_ca_trust/handlers/main.yml no-changed-when
+.ansible/collections/ansible_collections/devroles/system/roles/update_ca_trust/tasks/main.yml yaml[octal-values]
+.ansible/collections/ansible_collections/devroles/system/roles/webserver/defaults/main.yml var-naming[no-role-prefix]
+.ansible/collections/ansible_collections/devroles/system/roles/webserver/tasks/letsencrypt.yml args[module]
+galaxy.yml galaxy[no-changelog]
+galaxy.yml galaxy[tags]
+meta/runtime.yml meta-runtime[unsupported-version]
+playbooks/molecule/dotfiles/prepare.yml fqcn[action-core]
+playbooks/molecule/dotfiles/prepare.yml name[play]
+playbooks/molecule/sudo/molecule.yml yaml[line-length]
+roles/authorized_key/defaults/main.yml var-naming[no-role-prefix]
+roles/authorized_key/molecule/shared/playbook.yml name[casing]
+roles/authorized_key/molecule/shared/playbook.yml var-naming[no-role-prefix]
+roles/common/molecule/shared/playbook.yml name[casing]
+roles/container/molecule/shared/playbook.yml name[casing]
+roles/copy_or_link/molecule/default/playbook.yml name[casing]
+roles/copy_or_link/molecule/default/prepare.yml fqcn[action-core]
+roles/copy_or_link/molecule/default/prepare.yml name[casing]
+roles/copy_or_link/molecule/default/prepare.yml name[play]
+roles/copy_or_link/molecule/default/prepare.yml yaml[octal-values]
+roles/copy_or_link/molecule/local/playbook.yml fqcn[action-core]
+roles/copy_or_link/molecule/local/playbook.yml name[casing]
+roles/copy_or_link/molecule/local/prepare.yml fqcn[action-core]
+roles/copy_or_link/molecule/local/prepare.yml fqcn[action]
+roles/copy_or_link/molecule/local/prepare.yml name[casing]
+roles/copy_or_link/molecule/local/prepare.yml name[play]
+roles/copy_or_link/molecule/local/prepare.yml yaml[octal-values]
+roles/copy_or_link/molecule/local/remote_playbook.yml name[play]
+roles/dev/molecule/shared/playbook.yml name[casing]
+roles/dev/molecule/shared/prepare.yml name[casing]
+roles/dotfiles/molecule/shared/playbook.yml fqcn[action-core]
+roles/dotfiles/molecule/shared/playbook.yml name[casing]
+roles/dotfiles/tasks/03_vim.yml yaml[octal-values]
+roles/epel/molecule/shared/playbook.yml name[casing]
+roles/firewalld/defaults/main.yml yaml[comments]
+roles/firewalld/molecule/shared/playbook.yml name[casing]
+roles/flatpak/molecule/shared/playbook.yml fqcn[action-core]
+roles/mariadb/molecule/shared/playbook.yml fqcn[action-core]
+roles/mariadb/molecule/shared/playbook.yml name[casing]
+roles/mariadb/tasks/configure_access.yml yaml[octal-values]
+roles/mirror/molecule/shared/playbook.yml fqcn[action-core]
+roles/mirror/molecule/shared/playbook.yml name[casing]
+roles/mirror/molecule/shared/playbook.yml name[play]
+roles/mounts/defaults/main.yml var-naming[no-role-prefix]
+roles/mta/tasks/main.yml syntax-check[specific]
+roles/mythtv/handlers/main.yml no-changed-when
+roles/mythtv/tasks/main.yml yaml[octal-values]
+roles/packages_server/molecule/shared/playbook.yml name[casing]
+roles/packages_workstation/tasks/Linux/03_libvirt.yml yaml[octal-values]
+roles/packages_workstation/tasks/Linux/05_kerberos.yml yaml[octal-values]
+roles/packages_workstation/tasks/Linux/06_blu_ray.yml yaml[octal-values]
+roles/packages_workstation/vars/MacOSX.yml var-naming[no-role-prefix]
+roles/python_prep/molecule/shared/playbook.yml fqcn[action-core]
+roles/python_prep/molecule/shared/playbook.yml name[casing]
+roles/sudoers/defaults/main.yml var-naming[no-role-prefix]
+roles/sudoers/molecule/shared/playbook.yml name[casing]
+roles/sudoers/molecule/shared/prepare.yml fqcn[action-core]
+roles/sudoers/molecule/shared/prepare.yml name[casing]
+roles/sudoers/tasks/main.yml yaml[octal-values]
+roles/system_repositories/molecule/container/playbook.yml fqcn[action-core]
+roles/system_repositories/molecule/container/playbook.yml name[casing]
+roles/system_repositories/molecule/container_fedora/playbook.yml fqcn[action-core]
+roles/system_repositories/molecule/container_fedora/playbook.yml name[casing]
+roles/update_ca_trust/handlers/main.yml no-changed-when
+roles/update_ca_trust/molecule/shared/playbook.yml name[casing]
+roles/update_ca_trust/molecule/shared/prepare.yml fqcn[action-core]
+roles/update_ca_trust/molecule/shared/prepare.yml name[casing]
+roles/update_ca_trust/molecule/shared/prepare.yml yaml[octal-values]
+roles/update_ca_trust/tasks/main.yml yaml[octal-values]
+roles/webserver/defaults/main.yml var-naming[no-role-prefix]
+roles/webserver/tasks/letsencrypt.yml args[module]