diff --git a/.github/workflows/bindings-server.main.kts b/.github/workflows/bindings-server.main.kts
index cdf3daf03..d99634be1 100755
--- a/.github/workflows/bindings-server.main.kts
+++ b/.github/workflows/bindings-server.main.kts
@@ -123,21 +123,37 @@ workflow(
         runWithSpecificKotlinVersion(
             kotlinVersion = newestNotCompatibleVersion,
             command = """
-                cp .github/workflows/test-script-consuming-jit-bindings.main.kts .github/workflows/test-script-consuming-jit-bindings-too-old-kotlin.main.kts
-                ${failsWithPhraseInLogs(
-                    command = ".github/workflows/test-script-consuming-jit-bindings-too-old-kotlin.main.kts",
-                    // This test depicts the current behavior that the served bindings aren't
-                    // compatible with some older Kotlin version. We may want to address it one day.
-                    // For more info, see https://github.com/typesafegithub/github-workflows-kt/issues/1756
-                    phrase = "was compiled with an incompatible version of Kotlin",
-                )}
+                // This test depicts the current behavior that the served bindings aren't
+                // compatible with some older Kotlin version. We may want to address it one day.
+                // For more info, see https://github.com/typesafegithub/github-workflows-kt/issues/1756
+                val src = java.io.File(".github/workflows/test-script-consuming-jit-bindings.main.kts")
+                val dest = java.io.File(".github/workflows/test-script-consuming-jit-bindings-too-old-kotlin.main.kts")
+                src.copyTo(dest, overwrite = true).setExecutable(true)
+                val process = ProcessBuilder("kotlin", dest.path).redirectErrorStream(true).start()
+                val output = process.inputStream.bufferedReader().readText()
+                val exitCode = process.waitFor()
+                if (exitCode == 0) {
+                    System.err.println("Expected the script to fail but it succeeded")
+                    System.exit(1)
+                }
+                if (!output.contains("was compiled with an incompatible version of Kotlin")) {
+                    System.err.println("Expected error message not found. Output:")
+                    System.err.println(output)
+                    System.exit(1)
+                }
             """.trimIndent(),
         )
         runWithSpecificKotlinVersion(
             kotlinVersion = oldestCompatibleVersion,
             command = """
-                cp .github/workflows/test-script-consuming-jit-bindings.main.kts .github/workflows/test-script-consuming-jit-bindings-older-kotlin.main.kts
-                .github/workflows/test-script-consuming-jit-bindings-older-kotlin.main.kts
+                val src = java.io.File(".github/workflows/test-script-consuming-jit-bindings.main.kts")
+                val dest = java.io.File(".github/workflows/test-script-consuming-jit-bindings-older-kotlin.main.kts")
+                src.copyTo(dest, overwrite = true).setExecutable(true)
+                val exitCode = ProcessBuilder("kotlin", dest.path).inheritIO().start().waitFor()
+                if (exitCode != 0) {
+                    System.err.println("Script failed with exit code " + exitCode)
+                    System.exit(exitCode)
+                }
             """.trimIndent(),
         )
 
@@ -193,7 +209,10 @@ fun JobBuilder<JobOutputs.EMPTY>.cleanMavenLocal() {
     )
 }
 
-fun JobBuilder<JobOutputs.EMPTY>.runWithSpecificKotlinVersion(kotlinVersion: String, command: String) {
+fun JobBuilder<JobOutputs.EMPTY>.runWithSpecificKotlinVersion(
+    kotlinVersion: String,
+    command: String,
+) {
     uses(
         name = "Install Kotlin $kotlinVersion",
         action = SetupKotlin(
@@ -203,15 +222,12 @@ fun JobBuilder<JobOutputs.EMPTY>.runWithSpecificKotlinVersion(kotlinVersion: Str
     cleanMavenLocal()
     run(
         name = "Execute the script using the bindings from the server, using older Kotlin ($kotlinVersion) as consumer",
-        command = command,
+        command = """
+            cat > /tmp/script.kts << 'KOTLIN_SCRIPT'
+$command
+KOTLIN_SCRIPT
+            kotlin /tmp/script.kts
+        """.trimIndent(),
     )
 }
 
-fun failsWithPhraseInLogs(
-    command: String,
-    phrase: String,
-): String =
-   """
-       ($command || true) >> output.txt 2>&1
-       grep "$phrase" output.txt
-   """.trimIndent()
diff --git a/.github/workflows/bindings-server.yaml b/.github/workflows/bindings-server.yaml
index b343c308e..00133cde2 100644
--- a/.github/workflows/bindings-server.yaml
+++ b/.github/workflows/bindings-server.yaml
@@ -92,9 +92,27 @@ jobs:
     - id: 'step-10'
       name: 'Execute the script using the bindings from the server, using older Kotlin (1.9.0) as consumer'
       run: |2-
-                        cp .github/workflows/test-script-consuming-jit-bindings.main.kts .github/workflows/test-script-consuming-jit-bindings-too-old-kotlin.main.kts
-                        (.github/workflows/test-script-consuming-jit-bindings-too-old-kotlin.main.kts || true) >> output.txt 2>&1
-        grep "was compiled with an incompatible version of Kotlin" output.txt
+                    cat > /tmp/script.kts << 'KOTLIN_SCRIPT'
+        // This test depicts the current behavior that the served bindings aren't
+        // compatible with some older Kotlin version. We may want to address it one day.
+        // For more info, see https://github.com/typesafegithub/github-workflows-kt/issues/1756
+        val src = java.io.File(".github/workflows/test-script-consuming-jit-bindings.main.kts")
+        val dest = java.io.File(".github/workflows/test-script-consuming-jit-bindings-too-old-kotlin.main.kts")
+        src.copyTo(dest, overwrite = true).setExecutable(true)
+        val process = ProcessBuilder("kotlin", dest.path).redirectErrorStream(true).start()
+        val output = process.inputStream.bufferedReader().readText()
+        val exitCode = process.waitFor()
+        if (exitCode == 0) {
+            System.err.println("Expected the script to fail but it succeeded")
+            System.exit(1)
+        }
+        if (!output.contains("was compiled with an incompatible version of Kotlin")) {
+            System.err.println("Expected error message not found. Output:")
+            System.err.println(output)
+            System.exit(1)
+        }
+        KOTLIN_SCRIPT
+                    kotlin /tmp/script.kts
     - id: 'step-11'
       name: 'Install Kotlin 2.0.0'
       uses: 'fwilhe2/setup-kotlin@v1'
@@ -105,9 +123,18 @@ jobs:
       run: 'rm -rf ~/.m2/repository/'
     - id: 'step-13'
       name: 'Execute the script using the bindings from the server, using older Kotlin (2.0.0) as consumer'
-      run: |-
-        cp .github/workflows/test-script-consuming-jit-bindings.main.kts .github/workflows/test-script-consuming-jit-bindings-older-kotlin.main.kts
-        .github/workflows/test-script-consuming-jit-bindings-older-kotlin.main.kts
+      run: |2-
+                    cat > /tmp/script.kts << 'KOTLIN_SCRIPT'
+        val src = java.io.File(".github/workflows/test-script-consuming-jit-bindings.main.kts")
+        val dest = java.io.File(".github/workflows/test-script-consuming-jit-bindings-older-kotlin.main.kts")
+        src.copyTo(dest, overwrite = true).setExecutable(true)
+        val exitCode = ProcessBuilder("kotlin", dest.path).inheritIO().start().waitFor()
+        if (exitCode != 0) {
+            System.err.println("Script failed with exit code " + exitCode)
+            System.exit(exitCode)
+        }
+        KOTLIN_SCRIPT
+                    kotlin /tmp/script.kts
     - id: 'step-14'
       name: 'Compile a Gradle project using the bindings from the server'
       run: |-