fix: get changeset from rss

mob-sakai · mob-sakai · commit 7bd7d2f27e4e · 2024-11-05T11:12:25.000+09:00
diff --git a/.github/workflows/update-changesets.yml b/.github/workflows/update-changesets.yml
@@ -2,7 +2,6 @@ name: update-changesets
 
 env:
   WORKFLOW_FILE: update-changesets.yml
-  DB_FILE: db
   RSS_URLS: |
     https://unity.com/releases/editor/lts-releases.xml
     https://unity.com/releases/editor/tech-and-preview-releases.xml
@@ -39,30 +38,7 @@ jobs:
           [ "$RESULT" = "false" ] && echo "::error::Other '${{ env.WORKFLOW_FILE }}' workflows are in progress." && exit 1 || :
 
       - name: 🛠️ Update changesets file
-        run: |
-          touch "${{ env.DB_FILE }}"
-          urls=(`echo "${{ env.RSS_URLS }}"`)
-          for url in "${urls[@]}"; do
-            # Find versions from RSS
-            echo "Processing: $url" | tee -a $GITHUB_STEP_SUMMARY
-            versions=($(curl -s "$url" | grep -oE "${{ env.REGEX_UNITY_VERSION }}" | sort -u))
-
-            for version in "${versions[@]}"; do
-                # Already exists
-                if grep -q "$version" "${{ env.DB_FILE }}"; then
-                    continue
-                fi
-
-                # Get changeset for the Unity version
-                changeset_output=$(npx unity-changeset "$version" 2>/dev/null)
-
-                if [ $? -eq 0 ]; then
-                    echo -e "$version\t$changeset_output" | tee -a "${{ env.DB_FILE }}" $GITHUB_STEP_SUMMARY
-                else
-                    echo "  Error: $version is not valid"
-                fi
-            done
-          done
+        run: npx unity-changeset list --all > db
 
       - name: Commit & Push changes
         uses: actions-js/push@master
diff --git a/.vscode/launch.json b/.vscode/launch.json
@@ -4,6 +4,25 @@
     // 詳細情報は次を確認してください: https://go.microsoft.com/fwlink/?linkid=830387
     "version": "0.2.0",
     "configurations": [
+        {
+            "request": "launch",
+            "name": "list",
+            "type": "node",
+            "program": "${workspaceFolder}/src/cli.ts",
+            "cwd": "${workspaceFolder}",
+            "env": {},
+            "runtimeExecutable": "/Users/takashi.sakai/.asdf/shims/deno",
+            "runtimeArgs": [
+                "run",
+                "--unstable",
+                "--inspect-brk",
+                "--allow-all"
+            ],
+            "args": [
+                "list"
+            ],
+            "attachSimplePort": 9229
+        },
         {
             "request": "launch",
             "name": "list --versions",
@@ -43,6 +62,25 @@
             ],
             "attachSimplePort": 9229
         },
+        {
+            "request": "launch",
+            "name": "6000.0.25f2",
+            "type": "node",
+            "program": "${workspaceFolder}/src/cli.ts",
+            "cwd": "${workspaceFolder}",
+            "env": {},
+            "runtimeExecutable": "/Users/takashi.sakai/.asdf/shims/deno",
+            "runtimeArgs": [
+                "run",
+                "--unstable",
+                "--inspect-brk",
+                "--allow-all"
+            ],
+            "args": [
+                "6000.0.25f2"
+            ],
+            "attachSimplePort": 9229
+        },
         {
             "request": "launch",
             "name": "--help",
diff --git a/src/cli.ts b/src/cli.ts
@@ -30,9 +30,8 @@ new Command()
   .description("Find Unity changesets.")
   .example("unity-changeset 2018.4.36f1", "Get changeset of Unity 2018.4.36f1 ('6cd387d23174' will be output).")
   .arguments("<version>")
-  .globalOption("--db", "Database url", { default: "https://mob-sakai.github.io/unity-changeset/db" })
-  .action((options, version) => {
-    getUnityChangeset(options.db as string, version)
+  .action((_, version) => {
+    getUnityChangeset(version)
       .then((c) => console.log(c.changeset))
       .catch(() => {
         console.error("The given version was not found.");
@@ -46,7 +45,6 @@ new Command()
     "list",
     new Command()
       .description("List Unity changesets.")
-      .globalOption("--db", "Database url", { default: "https://mob-sakai.github.io/unity-changeset/db" })
       .example("unity-changeset list", "List changesets.")
       .example("unity-changeset list --all --json", "List changesets of all versions in json format.")
       .example("unity-changeset list --version-only --min 2018.3 --max 2019.4", "List all versions from 2018.3 to 2019.4.")
@@ -114,7 +112,7 @@ new Command()
             ? FormatMode.PrettyJson
             : FormatMode.None;
 
-        listChangesets(options.db as string, searchMode, filterOptions, groupMode, outputMode, formatMode)
+        listChangesets(searchMode, filterOptions, groupMode, outputMode, formatMode)
           .then((result) => console.log(result));
       }),
   )
diff --git a/src/index.ts b/src/index.ts
@@ -4,6 +4,15 @@ export const UnityChangeset = UnityChangesetClass;
 export type UnityChangeset = UnityChangesetClass;
 
 const REGEXP_HUB_LINKS = /unityhub:\/\/\d{4}\.\d+\.\d+(a|b|f)\d+\/\w{12}/g;
+const REGEXP_UNITY_VERSION = /\d{4}\.\d+\.\d+(a|b|f)\d+/g;
+const UNITY_CHANGESETS_DB_URL =
+  "https://mob-sakai.github.io/unity-changeset/db";
+const UNITY_RSS_URLS: string[] = [
+  "https://unity.com/releases/editor/lts-releases.xml",
+  "https://unity.com/releases/editor/tech-and-preview-releases.xml",
+  "https://unity.com/releases/editor/beta-releases.xml",
+  "https://unity.com/releases/editor/alpha-releases.xml",
+];
 
 /*
 Unity release URLs for each lifecycle.
@@ -21,67 +30,37 @@ const UNITY_RELEASE_URLS: { [key: string]: string } = {
  * @returns An Unity changeset.
  */
 export async function getUnityChangeset(
-  db: string,
   version: string,
 ): Promise<UnityChangeset> {
-  let results = await loadDb(db, version);
-  if (0 < results.length) {
-    return results[0];
-  }
+  const db = await loadDb();
+  const results = db.filter((c) => c.version === version);
+  return results.length > 0
+    ? results[0]
+    : getUnityChangesetFromReleasePage(version);
+}
 
-  const match = version.match(/^(\d\.\d+\.\d+)(a|b|f|p)\d+$/);
+async function getUnityChangesetFromReleasePage(
+  version: string,
+): Promise<UnityChangeset> {
+  const match = version.match(/^(\d+\.\d+\.\d+)(a|b|f|p)\d+$/);
   const lifecycle = match?.[2] as string;
   const releaseUrl = UNITY_RELEASE_URLS[lifecycle];
-  if (lifecycle == "f") {
-    const shortVersion = match?.[1] as string;
-    results = (await getUnityChangesetsFromUrl(releaseUrl + shortVersion))
-      .filter((c) => c.version === version);
-  } else {
-    results = (await getUnityChangesetsFromUrl(releaseUrl + version))
-      .filter((c) => c.version === version);
-  }
-
-  if (0 < results.length) {
-    return results[0];
-  }
-
-  throw new Error(`No changeset found for '${version}'`);
-}
-
-export async function loadDb(
-  db: string,
-  version?: string,
-): Promise<UnityChangeset[]> {
-  const response = await fetch(db);
+  const shortVersion = match?.[1] as string;
+  const releasePageUrl = releaseUrl +
+    (lifecycle == "f" ? shortVersion : version);
+  const response = await fetch(releasePageUrl);
   const text = await response.text();
-  const lines = text.split("\n");
-
-  if (version) {
-    const startsWith = version + "\t";
-    return lines
-      .filter((line) => line.startsWith(startsWith))
-      .map((line) => UnityChangeset.createFromDb(line));
-  } else {
-    return lines
-      .map((line) => UnityChangeset.createFromDb(line));
+  const matchLink = text.match(REGEXP_HUB_LINKS);
+  if (!matchLink) {
+    throw new Error(`No changeset found at '${releasePageUrl}'`);
   }
-}
 
-function getUnityChangesetsFromUrl(
-  url: string,
-): Promise<UnityChangeset[]> {
-  return fetch(url)
-    .then((response) => response.text())
-    .then((raw) => {
-      const match = raw.match(REGEXP_HUB_LINKS);
-
-      if (!match) {
-        console.error(`No changesets found at '${url}'`);
-        return [];
-      }
+  const changeset = UnityChangeset.createFromHref(matchLink[0]);
+  if (changeset.version !== version) {
+    throw new Error(`No changeset found at '${releasePageUrl}'`);
+  }
 
-      return match.map((m) => UnityChangeset.createFromHref(m));
-    });
+  return changeset;
 }
 
 /*
@@ -156,14 +135,13 @@ export enum FormatMode {
 }
 
 export function listChangesets(
-  db: string,
   searchMode: SearchMode,
   filterOptions: FilterOptions,
   groupMode: GroupMode,
   outputMode: OutputMode,
   formatMode: FormatMode,
 ): Promise<string> {
-  return searchChangesets(db, searchMode)
+  return searchChangesets(searchMode)
     .then((results) => filterChangesets(results, filterOptions))
     .then((results) => groupChangesets(results, groupMode))
     .then((results) => {
@@ -196,19 +174,48 @@ export function listChangesets(
     });
 }
 
+async function loadDb(): Promise<UnityChangeset[]> {
+  const response = await fetch(UNITY_CHANGESETS_DB_URL);
+  const text = await response.text();
+  const lines = text.split("\n");
+  return lines
+    .map((line) => UnityChangeset.createFromDb(line));
+}
+
+async function findVersions(url: string): Promise<string[]> {
+  const response = await fetch(url);
+  const text = await response.text();
+  const lines = text.split("\n");
+
+  const versions = lines
+    .map((l) => l.match(REGEXP_UNITY_VERSION))
+    .filter((m) => m)
+    .map((m) => m?.[0] as string);
+  return Array.from(new Set<string>(versions));
+}
+
 export async function searchChangesets(
-  db: string,
   searchMode: SearchMode,
 ): Promise<UnityChangeset[]> {
-  const results = await loadDb(db);
+  const results = await loadDb();
+  const versions = await Promise.all(UNITY_RSS_URLS
+    .map((url) => findVersions(url)));
+  const appendResults = await Promise.all(
+    versions
+      .flat()
+      .filter((v) => !results.some((r) => r.version === v))
+      .map((v) => getUnityChangesetFromReleasePage(v)),
+  );
+  const allResults = results.concat(appendResults);
+
   switch (searchMode) {
     case SearchMode.All:
-      return results;
+      return allResults;
     case SearchMode.Default:
-      return results
+      return allResults
         .filter((c) => !c.preRelease);
     case SearchMode.PreRelease:
-      return results
+      return allResults
         .filter((c) => c.preRelease);
     default:
       throw Error(`The given search mode '${searchMode}' was not supported`);
diff --git a/src/unityChangeset.test.ts b/src/unityChangeset.test.ts
@@ -14,8 +14,6 @@ import {
   UnityChangeset,
 } from "./index.ts";
 
-const DB_URL = "https://mob-sakai.github.io/unity-changeset/db";
-
 Deno.test("UnityChangeset.toNumber min", () => {
   assertEquals(UnityChangeset.toNumber("2018.3", false), 201803000000);
 });
@@ -34,22 +32,22 @@ Deno.test("UnityChangeset.toNumber max", () => {
 ].forEach((testcase) => {
   Deno.test(`getUnityChangeset (${testcase.version})`, async () => {
     if (testcase.expected) {
-      const changeset = (await getUnityChangeset(DB_URL, testcase.version)).changeset;
+      const changeset = (await getUnityChangeset(testcase.version)).changeset;
       assertEquals(changeset, testcase.expected);
     }
     else {
-      await assertRejects(() => getUnityChangeset(DB_URL, testcase.version));
+      await assertRejects(() => getUnityChangeset(testcase.version));
     }
   })
 });
 
 Deno.test("scrapeArchivedChangesets", async () => {
-  const changesets = await searchChangesets(DB_URL, SearchMode.Default);
+  const changesets = await searchChangesets(SearchMode.Default);
   assertNotEquals(changesets.length, 0);
 });
 
 Deno.test("scrapeBetaChangesets", async () => {
-  const changesets = await searchChangesets(DB_URL, SearchMode.PreRelease);
+  const changesets = await searchChangesets(SearchMode.PreRelease);
   console.log(changesets.map((c) => c.version));
   assertNotEquals(changesets.length, 0);
 });
@@ -61,7 +59,7 @@ Deno.test("scrapeBetaChangesets", async () => {
   { searchMode: SearchMode.PreRelease },
 ].forEach((testcase) => {
   Deno.test(`filterChangesets(${JSON.stringify(testcase.searchMode)})`, async () => {
-    const changesets = await searchChangesets(DB_URL, testcase.searchMode);
+    const changesets = await searchChangesets(testcase.searchMode);
     assertNotEquals(changesets.length, 0);
   });
 });
