diff --git a/bin/pa11y-ci.js b/bin/pa11y-ci.js
index 515b68bf..c2a11685 100755
--- a/bin/pa11y-ci.js
+++ b/bin/pa11y-ci.js
@@ -72,6 +72,7 @@ Promise.resolve()
 	.then(config => {
 		// Load a sitemap based on the `--sitemap` flag
 		if (commander.sitemap) {
+			config.urls = [];
 			return loadSitemapIntoConfig(commander, config);
 		}
 		return config;
@@ -209,6 +210,7 @@ function defaultConfig(config) {
 // Load a sitemap from a remote URL, parse out the
 // URLs, and add them to an existing config object
 function loadSitemapIntoConfig(program, initialConfig) {
+	console.log('config', initialConfig);
 	const sitemapFind = (
 		program.sitemapFind ?
 			new RegExp(program.sitemapFind, 'gi') :
diff --git a/test/integration/cli-sitemap.test.js b/test/integration/cli-sitemap.test.js
index 41fdfcb0..b48f1b90 100644
--- a/test/integration/cli-sitemap.test.js
+++ b/test/integration/cli-sitemap.test.js
@@ -107,3 +107,24 @@ describe('pa11y-ci (with a sitemap being sitemapindex)', () => {
 	});
 
 });
+
+describe('pa11y-ci (with a sitemap being sitemapindex) and config. should ignore config urls', () => {
+
+	before(() => {
+		return global.cliCall([
+			'--sitemap',
+			'http://localhost:8090/sitemapindex.xml',
+			'--config',
+			'extension-json'
+		]);
+	});
+
+	it('loads the expected urls from multiple sitemaps', () => {
+		assert.notInclude(global.lastResult.output, 'http://localhost:8090/config-extension-json');
+		assert.include(global.lastResult.output, 'http://localhost:8090/passing-1');
+		assert.include(global.lastResult.output, 'http://localhost:8090/failing-1');
+		assert.include(global.lastResult.output, 'http://localhost:8090/excluded');
+		assert.include(global.lastResult.output, 'http://localhost:8090/passing-2');
+	});
+
+});