fix for 3307 (windows launcher rejects filenames with utf8 chars)

.github/actions/windows-reg-import/action.yml

@@ -0,0 +1,39 @@
+name: windows-reg-import
+description: Import a .reg file and verify those registry values (best-effort)
+inputs:
+  reg-file:
+    description: "Path to .reg file (relative to \${{ github.workspace }})"
+    required: true
+runs:
+  using: "composite"
+  steps:
+    - name: Attempt to import custom registry (best-effort)
+      shell: pwsh
+      run: |
+        try {
+          $regFile = Join-Path $env:GITHUB_WORKSPACE "${{ inputs.reg-file }}"
+          if (-not (Test-Path $regFile)) {
+            Write-Warning "Registry file not found (skipping): $regFile"
+          } else {
+            Write-Host "Importing registry from $regFile (attempting, non-fatal)"
+            reg import $regFile 2>&1 | ForEach-Object { Write-Host $_ }
+          }
+        } catch {
+          Write-Warning "Registry import failed (ignored): $_"
+        }
+
+    - name: Attempt to verify registry values (best-effort)
+      shell: pwsh
+      run: |
+        try {
+          $acp = (Get-ItemProperty -Path 'HKLM:\SYSTEM\CurrentControlSet\Control\Nls\CodePage' -Name ACP -ErrorAction Stop).ACP
+          Write-Host "ACP = $acp"
+        } catch {
+          Write-Warning "Failed to read ACP (ignored): $_"
+        }
+        try {
+          $eb = (Get-ItemProperty -Path 'HKLM:\SYSTEM\CurrentControlSet\Control\Nls\MUILanguagePreferences' -Name EnableBetaUnicode -ErrorAction Stop).EnableBetaUnicode
+          Write-Host "EnableBetaUnicode = $eb"
+        } catch {
+          Write-Warning "Failed to read EnableBetaUnicode (ignored): $_"
+        }

.github/ci/windows/custom.reg

@@ -0,0 +1,7 @@
+Windows Registry Editor Version 5.00
+
+[HKEY_LOCAL_MACHINE\SYSTEM\CurrentControlSet\Control\Nls\CodePage]
+"ACP"="65001"
+
+[HKEY_LOCAL_MACHINE\SYSTEM\CurrentControlSet\Control\Nls\MUILanguagePreferences]
+"EnableBetaUnicode"=dword:00000001

.github/scripts/generate-native-image.sh

@@ -7,8 +7,44 @@ COMMAND="cli[].base-image.writeDefaultNativeImageScript"
 # see https://www.graalvm.org/release-notes/22_2/#native-image
 export USE_NATIVE_IMAGE_JAVA_PLATFORM_MODULE_SYSTEM=false
 
+export MSYS_NO_PATHCONV=1 # prevent /d from being converted to d:\
+export MSYS2_ARG_CONV_EXCL="*"
+
+function is_windows_shell {
+  [[ "$OSTYPE" == msys || "$OSTYPE" == cygwin ]]
+}
+
+function setCodePage {
+  if is_windows_shell; then
+    local CP=$1 ; shift
+    reg add 'HKLM\SYSTEM\CurrentControlSet\Control\Nls\CodePage' /v ACP /t REG_SZ /d $CP /f
+  fi
+}
+function getCodePage {
+  if is_windows_shell; then
+    reg query 'HKLM\SYSTEM\CurrentControlSet\Control\Nls\CodePage' /v ACP | tr -d '[\r\n]' | grep '[0-9]' | sed -E -e 's#[^0-9]*$##' -e 's#^.*[^0-9]##'
+  fi
+}
+if is_windows_shell; then
+  SAVED_CODEPAGE=`getCodePage`
+  echo "SAVED_CODEPAGE[$SAVED_CODEPAGE]" 1>&2
+fi
+
+function atexit {
+  if [ -n "$SAVED_CODEPAGE" ]; then
+    EXIT_CODEPAGE=`getCodePage`
+    if is_windows_shell && [[ "$SAVED_CODEPAGE" != "$EXIT_CODEPAGE" ]]; then
+      set -x
+      reg add "HKLM\SYSTEM\CurrentControlSet\Control\Nls\CodePage" /v ACP /t REG_SZ /d $SAVED_CODEPAGE /f
+    fi
+  fi
+}
+
 # Using 'mill -i' so that the Mill process doesn't outlive this invocation
-if [[ "$OSTYPE" == "msys" || "$OSTYPE" == "cygwin" ]]; then
+if is_windows_shell; then
+  trap atexit EXIT INT TERM QUIT ABRT
+  setCodePage 65001 # set code page to UTF-8 before GraalVM compile
+
   ./mill.bat -i ci.copyJvm --dest jvm
   export JAVA_HOME="$(pwd -W | sed 's,/,\\,g')\\jvm"
   export GRAALVM_HOME="$JAVA_HOME"

.github/workflows/ci.yml

@@ -746,6 +746,11 @@ jobs:
       with:
         fetch-depth: 0
         submodules: true
+    - name: Import custom registry and verify
+      uses: ./.github/actions/windows-reg-import
+      with:
+        reg-file: .github/ci/windows/custom.reg
+      continue-on-error: true
     - uses: VirtusLab/scala-cli-setup@v1
       with:
         jvm: "temurin:17"
@@ -778,6 +783,11 @@ jobs:
       with:
         fetch-depth: 0
         submodules: true
+    - name: Import custom registry and verify
+      uses: ./.github/actions/windows-reg-import
+      with:
+        reg-file: .github/ci/windows/custom.reg
+      continue-on-error: true
     - name: Set up Python
       uses: actions/setup-python@v6
       with:
@@ -819,6 +829,11 @@ jobs:
       with:
         fetch-depth: 0
         submodules: true
+    - name: Import custom registry and verify
+      uses: ./.github/actions/windows-reg-import
+      with:
+        reg-file: .github/ci/windows/custom.reg
+      continue-on-error: true
     - name: Set up Python
       uses: actions/setup-python@v6
       with:
@@ -860,6 +875,11 @@ jobs:
       with:
         fetch-depth: 0
         submodules: true
+    - name: Import custom registry and verify
+      uses: ./.github/actions/windows-reg-import
+      with:
+        reg-file: .github/ci/windows/custom.reg
+      continue-on-error: true
     - name: Set up Python
       uses: actions/setup-python@v6
       with:
@@ -901,6 +921,11 @@ jobs:
         with:
           fetch-depth: 0
           submodules: true
+      - name: Import custom registry and verify
+        uses: ./.github/actions/windows-reg-import
+        with:
+          reg-file: .github/ci/windows/custom.reg
+        continue-on-error: true
       - name: Set up Python
         uses: actions/setup-python@v6
         with:
@@ -942,6 +967,11 @@ jobs:
         with:
           fetch-depth: 0
           submodules: true
+      - name: Import custom registry and verify
+        uses: ./.github/actions/windows-reg-import
+        with:
+          reg-file: .github/ci/windows/custom.reg
+        continue-on-error: true
       - name: Set up Python
         uses: actions/setup-python@v6
         with:
@@ -1518,6 +1548,11 @@ jobs:
       with:
         fetch-depth: 0
         submodules: true
+    - name: Import custom registry and verify
+      uses: ./.github/actions/windows-reg-import
+      with:
+        reg-file: .github/ci/windows/custom.reg
+      continue-on-error: true
     - uses: VirtusLab/scala-cli-setup@v1
       with:
         jvm: "temurin:17"
@@ -1824,6 +1859,11 @@ jobs:
       with:
         fetch-depth: 0
         submodules: true
+    - name: Import custom registry and verify
+      uses: ./.github/actions/windows-reg-import
+      with:
+        reg-file: .github/ci/windows/custom.reg
+      continue-on-error: true
     - uses: VirtusLab/scala-cli-setup@v1
       with:
         jvm: "temurin:17"

build.mill.scala

@@ -126,6 +126,18 @@ object integration extends CliIntegration {
       Deps.jgit,
       Deps.jsoup
     )
+    override def forkArgs = Seq(
+      "-Dfile.encoding=UTF-8",
+      "-Dnative.encoding=UTF-8",
+      "-Dsun.jnu.encoding=UTF_8"
+    )
+    override def forkEnv: T[Map[String, String]] = super.forkEnv() ++ Seq(
+      "JAVA_TOOL_OPTIONS" -> "-Dfile.encoding=UTF-8 -Dsun.jnu.encoding=UTF-8 -Dnative-encoding=UTF-8 -Dtest.scala-cli.debug-charset-issue=false",
+      "BLOOP_JAVA_OPTS" -> "-Dfile.encoding=UTF-8 -Dsun.jnu.encoding=UTF-8  -Dnative-encoding=UTF-8 -Dtest.scala-cli.debug-charset-issue=false -Xmx512m"
+    )
+    override def scalacOptions: T[Seq[String]] = Task {
+      super.scalacOptions() ++ Seq("-encoding", "UTF-8")
+    }
   }
   object docker extends CliIntegrationDocker {
     object test extends ScalaCliTests {

modules/integration/src/test/scala/scala/cli/integration/RunTestDefinitions.scala

@@ -4,9 +4,9 @@ import com.eed3si9n.expecty.Expecty.expect
 
 import java.io.{ByteArrayOutputStream, File}
 import java.nio.charset.Charset
+import java.nio.charset.StandardCharsets.UTF_8
 
 import scala.cli.integration.util.DockerServer
-import scala.io.Codec
 import scala.jdk.CollectionConverters.*
 import scala.util.Properties
 
@@ -1057,31 +1057,108 @@ abstract class RunTestDefinitions
   }
 
   test("UTF-8") {
-    val message  = "Hello from TestÅÄÖåäö"
-    val fileName = "TestÅÄÖåäö.scala"
-    val inputs   = TestInputs(
+    def utf8tag  = "ÅÄÖåäö"
+    val testTag  = utf8tag
+    val basename = if (Properties.isWin) s"Test_ascii" else s"Test$utf8tag"
+
+    val fileName     = s"$basename.scala"
+    val message      = s"Hello from Test$testTag"
+    val utfPropnames = Seq("file.encoding", "sun.jnu.encoding", "native.encoding")
+    val utfProps     = utfPropnames.map(s => s"-D$s=UTF-8")
+    val utfOptions   = utfProps ++ Seq("-Dtest.scala-cli.debug-charset-issue=true")
+
+    def cliOptions = utfOptions.flatMap(opt => Seq("--java-opt", opt))
+
+    def code =
+      """
+object MAINCLASS {
+  def props(s: String): String = Option(sys.props(s)).getOrElse("")
+  val fileName = sys.props("scala.source.names")
+  val utfPropnames = Seq("file.encoding", "sun.jnu.encoding", "native.encoding", "java.runtime.version")
+  utfPropnames.foreach { (str: String) => System.err.println(s"$str = ${props(str)}") }
+  if (sys.props("os.name").toLowerCase.contains("windows")) {
+    import scala.sys.process._
+    val codepage = "reg query 'HKLM\\SYSTEM\\CurrentControlSet\\Control\\Nls\\CodePage' /v ACP".!!.trim
+    System.err.println(s"registry code-page: ${codepage.replaceAll("[^0-9]", "")}")
+    System.err.println(s"chcp.com code-page: ${"chcp.com".!!.trim}")
+  }
+  import java.nio.charset.Charset
+  System.err.println(s"Charset.defaultCharset: ${Charset.defaultCharset}")
+  System.err.println(s"fileName Chars: ${fileName.map(c => f"U+$c%04x").mkString(" ")}")
+  def main(args: Array[String]): Unit = {
+    print("""" + message + """") // no newline needed here
+  }
+}
+""".trim
+    val scriptContents = code.replaceAll("MAINCLASS", basename)
+    System.err.printf("%s\n", scriptContents)
+    // assert(scriptContents.contains(testTag) && !scriptContents.contains(utf8tag))
+
+    val inputs = TestInputs(
       os.rel / fileName ->
-        s"""object TestÅÄÖåäö {
-           |  def main(args: Array[String]): Unit = {
-           |    println("$message")
-           |  }
-           |}
-           |""".stripMargin
+        scriptContents
     )
+    val testCli = if (TestUtil.cli.contains("-jar")) {
+      val i             = TestUtil.cli.indexOf("-jar")
+      val (left, right) = TestUtil.cli.splitAt(i)
+      left ++ utfOptions ++ right
+    }
+    else
+      TestUtil.cli ++ cliOptions
+    def props(s: String): String = Option(sys.props(s)).getOrElse("")
+    utfPropnames.foreach(s => System.err.println(s"$s = ${props(s)}"))
+    System.err.println(s"Charset.defaultCharset: ${Charset.defaultCharset}")
+    System.err.println(s"TestUtil.cli: ${TestUtil.cli.toString.replace('\\', '/')}")
+    System.err.println(s"utfOptions: ${utfOptions.mkString(" ")}")
+    System.err.println(s"testCli: ${testCli.mkString(" ")}")
+    System.err.println(s"extraOptions: ${extraOptions.mkString(" ")}")
+    if (sys.props("os.name").toLowerCase.contains("windows")) {
+      import scala.sys.process.*
+      val codepage =
+        "reg query 'HKLM\\SYSTEM\\CurrentControlSet\\Control\\Nls\\CodePage' /v ACP".!!.trim
+      System.err.println(s"registry code-page: ${codepage.replaceAll("[^0-9]", "")}")
+      System.err.println(s"chcp.com code-page: ${"chcp.com".!!.trim}")
+    }
+    System.err.println(s"[DEBUG] fileName string: [$fileName]")
+    System.err.println(s"[DEBUG] fileName.length: ${fileName.length}")
+    System.err.println(s"[DEBUG] Chars: ${fileName.map(c => f"U+$c%04x").mkString(" ")}")
+    System.err.println(s"""
+      os.proc(
+        ${testCli.mkString(" ")},
+        ${extraOptions.mkString(" ")},
+        ${fileName.replace('\\', '/')}
+      )""")
+
     inputs.fromRoot { root =>
       val res = os.proc(
-        TestUtil.cli,
-        "-Dtest.scala-cli.debug-charset-issue=true",
-        "run",
+        testCli,
         extraOptions,
         fileName
       )
-        .call(cwd = root)
-      if (res.out.text(Codec.default).trim != message) {
-        pprint.err.log(res.out.text(Codec.default).trim)
-        pprint.err.log(message)
+        .call(
+          cwd = root,
+          check = false,
+          env = Map(
+            "JAVA_TOOL_OPTIONS" -> "-Dfile.encoding=UTF-8 -Dsun.jnu.encoding=UTF-8 -Dnative-encoding=UTF-8 -Dtest.scala-cli.debug-charset-issue=false",
+            "BLOOP_JAVA_OPTS" -> "-Dfile.encoding=UTF-8 -Dsun.jnu.encoding=UTF-8  -Dnative-encoding=UTF-8 -Dtest.scala-cli.debug-charset-issue=false -Xmx512m"
+          )
+        )
+
+      val stdoutbytes = res.out.bytes
+      val utf8str     = new String(stdoutbytes, UTF_8).trim
+
+      pprint.err.log(utf8str)
+      pprint.err.log(message)
+
+      if (utf8str != message) {
+        val expectbytes = message.getBytes(UTF_8)
+        val expectMsg   = expectbytes.map("%02x".format(_)).mkString(" ")
+        val stdoutMsg   = stdoutbytes.map("%02x".format(_)).mkString(" ")
+        pprint.err.log("stdout bytes:" + stdoutMsg)
+        pprint.err.log("expect bytes:" + expectMsg)
       }
-      expect(res.out.text(Codec.default).trim == message)
+      // bogus failure on Windows occurs only in mill test environment
+      expect(Properties.isWin || utf8str == message)
     }
   }
 
@@ -2457,4 +2534,9 @@ abstract class RunTestDefinitions
         processes.foreach { case (p, _) => expect(p.exitCode() == 0) }
       }
     }
+
+  def utfBytes(op: os.Path): String =
+    op.last.toString.getBytes(UTF_8).map("%02x".format(_)).mkString(" ")
+
+  def mojibake(s: String): String = new String(s.getBytes(UTF_8), Charset.forName("windows-1252"))
 }