ci(hf): use temporary repos per CI job to avoid concurrent commit races

.github/services/hf/huggingface/action.yml → .github/actions/hf-temp-repo/action.yml

@@ -15,19 +15,21 @@
 # specific language governing permissions and limitations
 # under the License.
 
-name: huggingface
-description: "Behavior test for Huggingface File System"
+name: "HF Temp Repo"
+description: "Create a temporary HuggingFace repo or bucket for testing, automatically deleted after the job"
+inputs:
+  repo_id:
+    description: "Full repo/bucket ID (e.g. org/repo-name)"
+    required: true
+  repo_type:
+    description: "Repository type (dataset, model, space, bucket)"
+    required: true
+  token:
+    description: "HuggingFace API token"
+    required: true
 
 runs:
-  using: "composite"
-  steps:
-    - name: Setup
-      shell: bash
-      run: |
-        cat << EOF >> $GITHUB_ENV
-        OPENDAL_HF_REPO_TYPE=dataset
-        OPENDAL_HF_REPO_ID=opendal/huggingface-testdata
-        OPENDAL_HF_REVISION=main
-        OPENDAL_HF_ROOT=/testdata/
-        OPENDAL_DISABLE_RANDOM_ROOT=true
-        EOF
+  using: "node24"
+  main: "setup.js"
+  post: "cleanup.js"
+  post-if: "always()"

.github/actions/hf-temp-repo/cleanup.js

@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+const { hfRequest, getState } = require("./common");
+
+async function run() {
+  const repoId = getState("repo_id");
+  const repoType = getState("repo_type");
+  const token = getState("token");
+
+  if (!repoId) {
+    console.log("No temp repo to clean up");
+    return;
+  }
+
+  if (repoType === "bucket") {
+    await hfRequest("DELETE", `/api/buckets/${repoId}`, token);
+  } else {
+    await hfRequest("DELETE", "/api/repos/delete", token, {
+      type: repoType,
+      name: repoId,
+    });
+  }
+
+  console.log(`Deleted temp ${repoType}: ${repoId}`);
+}
+
+run().catch((err) => console.warn(`Cleanup failed: ${err.message}`));

.github/actions/hf-temp-repo/common.js

@@ -0,0 +1,71 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+const fs = require("fs");
+const https = require("https");
+
+function hfRequest(method, path, token, body) {
+  return new Promise((resolve, reject) => {
+    const data = body ? JSON.stringify(body) : null;
+    const req = https.request(
+      {
+        hostname: "huggingface.co",
+        path,
+        method,
+        headers: {
+          Authorization: `Bearer ${token}`,
+          ...(data && {
+            "Content-Type": "application/json",
+            "Content-Length": Buffer.byteLength(data),
+          }),
+        },
+      },
+      (res) => {
+        let result = "";
+        res.on("data", (chunk) => (result += chunk));
+        res.on("end", () =>
+          res.statusCode >= 200 && res.statusCode < 300
+            ? resolve(result)
+            : reject(new Error(`HTTP ${res.statusCode}: ${result}`))
+        );
+      }
+    );
+    req.on("error", reject);
+    if (data) req.write(data);
+    req.end();
+  });
+}
+
+function getInput(name) {
+  return process.env[`INPUT_${name.toUpperCase()}`] || "";
+}
+
+function getState(name) {
+  return process.env[`STATE_${name}`] || "";
+}
+
+function saveState(name, value) {
+  fs.appendFileSync(process.env.GITHUB_STATE, `${name}=${value}\n`);
+}
+
+function exportVariable(name, value) {
+  fs.appendFileSync(process.env.GITHUB_ENV, `${name}=${value}\n`);
+}
+
+module.exports = { hfRequest, getInput, getState, saveState, exportVariable };

.github/actions/hf-temp-repo/setup.js

@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+const { hfRequest, getInput, saveState, exportVariable } = require("./common");
+
+async function run() {
+  const repoId = getInput("repo_id");
+  const repoType = getInput("repo_type");
+  const token = getInput("token");
+  const [organization, repoName] = repoId.split("/");
+
+  if (repoType === "bucket") {
+    await hfRequest("POST", "/api/buckets", token, {
+      name: repoName,
+      organization,
+      private: true,
+    });
+  } else {
+    await hfRequest("POST", "/api/repos/create", token, {
+      type: repoType,
+      name: repoName,
+      organization,
+      private: true,
+    });
+  }
+
+  console.log(`Created temp ${repoType}: ${repoId}`);
+  saveState("repo_id", repoId);
+  saveState("repo_type", repoType);
+  saveState("token", token);
+  exportVariable("OPENDAL_HF_REPO_ID", repoId);
+}
+
+run().catch((err) => {
+  console.error(err.message);
+  process.exit(1);
+});

.github/services/hf/hf_bucket/action.yml

@@ -27,7 +27,12 @@ runs:
         export-env: true
       env:
         OPENDAL_HF_TOKEN: op://services/hf/token
-        OPENDAL_HF_REPO_ID: op://services/hf/bucket_repo_id
+    - name: Create temp bucket
+      uses: ./.github/actions/hf-temp-repo
+      with:
+        repo_id: opendal/test-bucket-${{ github.run_id }}-${{ github.job }}
+        repo_type: bucket
+        token: ${{ env.OPENDAL_HF_TOKEN }}
     - name: Setup env
       shell: bash
       run: |

.github/services/hf/hf_dataset/action.yml

@@ -27,7 +27,12 @@ runs:
         export-env: true
       env:
         OPENDAL_HF_TOKEN: op://services/hf/token
-        OPENDAL_HF_REPO_ID: op://services/hf/dataset_repo_id
+    - name: Create temp dataset repo
+      uses: ./.github/actions/hf-temp-repo
+      with:
+        repo_id: opendal/test-dataset-${{ github.run_id }}-${{ github.job }}
+        repo_type: dataset
+        token: ${{ env.OPENDAL_HF_TOKEN }}
     - name: Setup env
       shell: bash
       run: |

core/services/hf/src/lib.rs

@@ -62,3 +62,4 @@ mod tests {
         assert_eq!(HUGGINGFACE_SCHEME, "huggingface");
     }
 }
+