From 2c7df0eaf8f5d3db66123d67e86b7c67cd68de48 Mon Sep 17 00:00:00 2001 From: ranjanash Date: Sat, 17 May 2025 17:51:19 +0530 Subject: [PATCH 1/3] Fix UTs for TestCombinedSystemMetricsPublisher --- .../TestCombinedSystemMetricsPublisher.java | 27 ++++++++++--------- 1 file changed, 15 insertions(+), 12 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestCombinedSystemMetricsPublisher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestCombinedSystemMetricsPublisher.java index 49baa0cb3616d..acd4095ee88d8 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestCombinedSystemMetricsPublisher.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestCombinedSystemMetricsPublisher.java @@ -194,6 +194,13 @@ private static YarnConfiguration getConf(boolean v1Enabled, boolean v2Enabled) { YarnConfiguration yarnConf = new YarnConfiguration(); + // Always set memory store implementations regardless of whether service is enabled + yarnConf.setClass(YarnConfiguration.TIMELINE_SERVICE_STORE, + MemoryTimelineStore.class, TimelineStore.class); + yarnConf.setClass(YarnConfiguration.TIMELINE_SERVICE_STATE_STORE_CLASS, + MemoryTimelineStateStore.class, TimelineStateStore.class); + + if (v1Enabled || v2Enabled) { yarnConf.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, true); } else { @@ -202,10 +209,6 @@ private static YarnConfiguration getConf(boolean v1Enabled, if (v1Enabled) { yarnConf.set(YarnConfiguration.TIMELINE_SERVICE_VERSION, "1.0"); - yarnConf.setClass(YarnConfiguration.TIMELINE_SERVICE_STORE, - MemoryTimelineStore.class, TimelineStore.class); - yarnConf.setClass(YarnConfiguration.TIMELINE_SERVICE_STATE_STORE_CLASS, - MemoryTimelineStateStore.class, TimelineStateStore.class); } if (v2Enabled) { @@ -444,7 +447,7 @@ private static RMAppAttempt createRMAppAttempt(boolean unmanagedAMAttempt) { RMAppAttempt appAttempt = mock(RMAppAttempt.class); when(appAttempt.getAppAttemptId()).thenReturn(appAttemptId); when(appAttempt.getHost()).thenReturn("test host"); - when(appAttempt.getRpcPort()).thenReturn(-100); + when(appAttempt.getRpcPort()).thenReturn(Integer.valueOf(-100)); if (!unmanagedAMAttempt) { Container container = mock(Container.class); when(container.getId()) @@ -465,9 +468,9 @@ private static RMApp createRMApp(ApplicationId appId) { when(rmApp.getApplicationType()).thenReturn("test app type"); when(rmApp.getUser()).thenReturn("testUser"); when(rmApp.getQueue()).thenReturn("test queue"); - when(rmApp.getSubmitTime()).thenReturn(Integer.MAX_VALUE + 1L); - when(rmApp.getStartTime()).thenReturn(Integer.MAX_VALUE + 2L); - when(rmApp.getFinishTime()).thenReturn(Integer.MAX_VALUE + 3L); + when(rmApp.getSubmitTime()).thenReturn(Long.valueOf(Integer.MAX_VALUE + 1L)); + when(rmApp.getStartTime()).thenReturn(Long.valueOf(Integer.MAX_VALUE + 2L)); + when(rmApp.getFinishTime()).thenReturn(Long.valueOf(Integer.MAX_VALUE + 3L)); when(rmApp.getDiagnostics()).thenReturn( new StringBuilder("test diagnostics info")); RMAppAttempt appAttempt = mock(RMAppAttempt.class); @@ -478,12 +481,12 @@ private static RMApp createRMApp(ApplicationId appId) { FinalApplicationStatus.UNDEFINED); Map resourceMap = new HashMap<>(); resourceMap - .put(ResourceInformation.MEMORY_MB.getName(), (long) Integer.MAX_VALUE); - resourceMap.put(ResourceInformation.VCORES.getName(), Long.MAX_VALUE); + .put(ResourceInformation.MEMORY_MB.getName(), Long.valueOf((long) Integer.MAX_VALUE)); + resourceMap.put(ResourceInformation.VCORES.getName(), Long.valueOf(Long.MAX_VALUE)); Map preemptedMap = new HashMap<>(); preemptedMap - .put(ResourceInformation.MEMORY_MB.getName(), (long) Integer.MAX_VALUE); - preemptedMap.put(ResourceInformation.VCORES.getName(), Long.MAX_VALUE); + .put(ResourceInformation.MEMORY_MB.getName(), Long.valueOf((long) Integer.MAX_VALUE)); + preemptedMap.put(ResourceInformation.VCORES.getName(), Long.valueOf(Long.MAX_VALUE)); when(rmApp.getRMAppMetrics()).thenReturn( new RMAppMetrics(Resource.newInstance(0, 0), 0, 0, resourceMap, preemptedMap, 0)); From 168b7f92b9aa436e0053d98c4a634eeabec0b92f Mon Sep 17 00:00:00 2001 From: ranjanash Date: Sat, 17 May 2025 17:56:40 +0530 Subject: [PATCH 2/3] Fix UTs for TestCombinedSystemMetricsPublisher --- .../TestCombinedSystemMetricsPublisher.java | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestCombinedSystemMetricsPublisher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestCombinedSystemMetricsPublisher.java index acd4095ee88d8..bba8ce858d504 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestCombinedSystemMetricsPublisher.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestCombinedSystemMetricsPublisher.java @@ -447,7 +447,7 @@ private static RMAppAttempt createRMAppAttempt(boolean unmanagedAMAttempt) { RMAppAttempt appAttempt = mock(RMAppAttempt.class); when(appAttempt.getAppAttemptId()).thenReturn(appAttemptId); when(appAttempt.getHost()).thenReturn("test host"); - when(appAttempt.getRpcPort()).thenReturn(Integer.valueOf(-100)); + when(appAttempt.getRpcPort()).thenReturn(-100); if (!unmanagedAMAttempt) { Container container = mock(Container.class); when(container.getId()) @@ -468,9 +468,9 @@ private static RMApp createRMApp(ApplicationId appId) { when(rmApp.getApplicationType()).thenReturn("test app type"); when(rmApp.getUser()).thenReturn("testUser"); when(rmApp.getQueue()).thenReturn("test queue"); - when(rmApp.getSubmitTime()).thenReturn(Long.valueOf(Integer.MAX_VALUE + 1L)); - when(rmApp.getStartTime()).thenReturn(Long.valueOf(Integer.MAX_VALUE + 2L)); - when(rmApp.getFinishTime()).thenReturn(Long.valueOf(Integer.MAX_VALUE + 3L)); + when(rmApp.getSubmitTime()).thenReturn(Integer.MAX_VALUE + 1L); + when(rmApp.getStartTime()).thenReturn(Integer.MAX_VALUE + 2L); + when(rmApp.getFinishTime()).thenReturn(Integer.MAX_VALUE + 3L); when(rmApp.getDiagnostics()).thenReturn( new StringBuilder("test diagnostics info")); RMAppAttempt appAttempt = mock(RMAppAttempt.class); @@ -481,12 +481,12 @@ private static RMApp createRMApp(ApplicationId appId) { FinalApplicationStatus.UNDEFINED); Map resourceMap = new HashMap<>(); resourceMap - .put(ResourceInformation.MEMORY_MB.getName(), Long.valueOf((long) Integer.MAX_VALUE)); - resourceMap.put(ResourceInformation.VCORES.getName(), Long.valueOf(Long.MAX_VALUE)); + .put(ResourceInformation.MEMORY_MB.getName(), (long) Integer.MAX_VALUE); + resourceMap.put(ResourceInformation.VCORES.getName(), Long.MAX_VALUE); Map preemptedMap = new HashMap<>(); preemptedMap - .put(ResourceInformation.MEMORY_MB.getName(), Long.valueOf((long) Integer.MAX_VALUE)); - preemptedMap.put(ResourceInformation.VCORES.getName(), Long.valueOf(Long.MAX_VALUE)); + .put(ResourceInformation.MEMORY_MB.getName(), (long) Integer.MAX_VALUE); + preemptedMap.put(ResourceInformation.VCORES.getName(), Long.MAX_VALUE); when(rmApp.getRMAppMetrics()).thenReturn( new RMAppMetrics(Resource.newInstance(0, 0), 0, 0, resourceMap, preemptedMap, 0)); From b2b1963bc49e11e2076577340149bfec83f5295d Mon Sep 17 00:00:00 2001 From: ranjanash Date: Mon, 26 May 2025 19:02:43 +0530 Subject: [PATCH 3/3] Added docker command to download and install visual C++ 2010 Redistributable package --- dev-support/docker/Dockerfile_windows_10 | 5 +++++ .../metrics/TestCombinedSystemMetricsPublisher.java | 11 ++++------- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/dev-support/docker/Dockerfile_windows_10 b/dev-support/docker/Dockerfile_windows_10 index bbd571ae62f35..e0ed446abd0f3 100644 --- a/dev-support/docker/Dockerfile_windows_10 +++ b/dev-support/docker/Dockerfile_windows_10 @@ -119,6 +119,11 @@ RUN powershell pkg-resolver\install-python.ps1 RUN powershell pkg-resolver\install-pip.ps1 RUN powershell pip install python-dateutil +## Install the Microsoft Visual C++ 2010 Redistributable to link leveldbjni native library +RUN powershell -Command Invoke-WebRequest -Uri https://download.microsoft.com/download/1/6/5/165255E7-1014-4D0A-B094-B6A430A6BFFC/vcredist_x64.exe -OutFile vcredist_x64.exe +RUN powershell -Command Start-Process -FilePath .\vcredist_x64.exe -ArgumentList '/quiet', '/norestart' -Wait +RUN powershell -Command Remove-Item vcredist_x64.exe + # Create a user HadoopBuilder with basic privileges and use it for building Hadoop on Windows. RUN powershell New-LocalUser -Name 'HadoopBuilder' -Description 'User account for building Apache Hadoop' -Password ([securestring]::new()) -AccountNeverExpires -PasswordNeverExpires diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestCombinedSystemMetricsPublisher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestCombinedSystemMetricsPublisher.java index bba8ce858d504..49baa0cb3616d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestCombinedSystemMetricsPublisher.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestCombinedSystemMetricsPublisher.java @@ -194,13 +194,6 @@ private static YarnConfiguration getConf(boolean v1Enabled, boolean v2Enabled) { YarnConfiguration yarnConf = new YarnConfiguration(); - // Always set memory store implementations regardless of whether service is enabled - yarnConf.setClass(YarnConfiguration.TIMELINE_SERVICE_STORE, - MemoryTimelineStore.class, TimelineStore.class); - yarnConf.setClass(YarnConfiguration.TIMELINE_SERVICE_STATE_STORE_CLASS, - MemoryTimelineStateStore.class, TimelineStateStore.class); - - if (v1Enabled || v2Enabled) { yarnConf.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, true); } else { @@ -209,6 +202,10 @@ private static YarnConfiguration getConf(boolean v1Enabled, if (v1Enabled) { yarnConf.set(YarnConfiguration.TIMELINE_SERVICE_VERSION, "1.0"); + yarnConf.setClass(YarnConfiguration.TIMELINE_SERVICE_STORE, + MemoryTimelineStore.class, TimelineStore.class); + yarnConf.setClass(YarnConfiguration.TIMELINE_SERVICE_STATE_STORE_CLASS, + MemoryTimelineStateStore.class, TimelineStateStore.class); } if (v2Enabled) {