diff --git a/.github/services/hf/huggingface/action.yml b/.github/actions/hf-temp-repo/action.yml similarity index 62% rename from .github/services/hf/huggingface/action.yml rename to .github/actions/hf-temp-repo/action.yml index 1174a930a7bc..4a3401107688 100644 --- a/.github/services/hf/huggingface/action.yml +++ b/.github/actions/hf-temp-repo/action.yml @@ -15,19 +15,21 @@ # specific language governing permissions and limitations # under the License. -name: huggingface -description: "Behavior test for Huggingface File System" +name: "HF Temp Repo" +description: "Create a temporary HuggingFace repo or bucket for testing, automatically deleted after the job" +inputs: + repo_id: + description: "Full repo/bucket ID (e.g. org/repo-name)" + required: true + repo_type: + description: "Repository type (dataset, model, space, bucket)" + required: true + token: + description: "HuggingFace API token" + required: true runs: - using: "composite" - steps: - - name: Setup - shell: bash - run: | - cat << EOF >> $GITHUB_ENV - OPENDAL_HF_REPO_TYPE=dataset - OPENDAL_HF_REPO_ID=opendal/huggingface-testdata - OPENDAL_HF_REVISION=main - OPENDAL_HF_ROOT=/testdata/ - OPENDAL_DISABLE_RANDOM_ROOT=true - EOF + using: "node24" + main: "setup.js" + post: "cleanup.js" + post-if: "always()" diff --git a/.github/actions/hf-temp-repo/cleanup.js b/.github/actions/hf-temp-repo/cleanup.js new file mode 100644 index 000000000000..48073d5a7a42 --- /dev/null +++ b/.github/actions/hf-temp-repo/cleanup.js @@ -0,0 +1,44 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +const { hfRequest, getState } = require("./common"); + +async function run() { + const repoId = getState("repo_id"); + const repoType = getState("repo_type"); + const token = getState("token"); + + if (!repoId) { + console.log("No temp repo to clean up"); + return; + } + + if (repoType === "bucket") { + await hfRequest("DELETE", `/api/buckets/${repoId}`, token); + } else { + await hfRequest("DELETE", "/api/repos/delete", token, { + type: repoType, + name: repoId, + }); + } + + console.log(`Deleted temp ${repoType}: ${repoId}`); +} + +run().catch((err) => console.warn(`Cleanup failed: ${err.message}`)); diff --git a/.github/actions/hf-temp-repo/common.js b/.github/actions/hf-temp-repo/common.js new file mode 100644 index 000000000000..7dfc33b36820 --- /dev/null +++ b/.github/actions/hf-temp-repo/common.js @@ -0,0 +1,71 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +const fs = require("fs"); +const https = require("https"); + +function hfRequest(method, path, token, body) { + return new Promise((resolve, reject) => { + const data = body ? JSON.stringify(body) : null; + const req = https.request( + { + hostname: "huggingface.co", + path, + method, + headers: { + Authorization: `Bearer ${token}`, + ...(data && { + "Content-Type": "application/json", + "Content-Length": Buffer.byteLength(data), + }), + }, + }, + (res) => { + let result = ""; + res.on("data", (chunk) => (result += chunk)); + res.on("end", () => + res.statusCode >= 200 && res.statusCode < 300 + ? resolve(result) + : reject(new Error(`HTTP ${res.statusCode}: ${result}`)) + ); + } + ); + req.on("error", reject); + if (data) req.write(data); + req.end(); + }); +} + +function getInput(name) { + return process.env[`INPUT_${name.toUpperCase()}`] || ""; +} + +function getState(name) { + return process.env[`STATE_${name}`] || ""; +} + +function saveState(name, value) { + fs.appendFileSync(process.env.GITHUB_STATE, `${name}=${value}\n`); +} + +function exportVariable(name, value) { + fs.appendFileSync(process.env.GITHUB_ENV, `${name}=${value}\n`); +} + +module.exports = { hfRequest, getInput, getState, saveState, exportVariable }; diff --git a/.github/actions/hf-temp-repo/setup.js b/.github/actions/hf-temp-repo/setup.js new file mode 100644 index 000000000000..3be5f938c5b1 --- /dev/null +++ b/.github/actions/hf-temp-repo/setup.js @@ -0,0 +1,53 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +const { hfRequest, getInput, saveState, exportVariable } = require("./common"); + +async function run() { + const repoId = getInput("repo_id"); + const repoType = getInput("repo_type"); + const token = getInput("token"); + const [organization, repoName] = repoId.split("/"); + + if (repoType === "bucket") { + await hfRequest("POST", "/api/buckets", token, { + name: repoName, + organization, + private: true, + }); + } else { + await hfRequest("POST", "/api/repos/create", token, { + type: repoType, + name: repoName, + organization, + private: true, + }); + } + + console.log(`Created temp ${repoType}: ${repoId}`); + saveState("repo_id", repoId); + saveState("repo_type", repoType); + saveState("token", token); + exportVariable("OPENDAL_HF_REPO_ID", repoId); +} + +run().catch((err) => { + console.error(err.message); + process.exit(1); +}); diff --git a/.github/services/hf/hf_bucket/action.yml b/.github/services/hf/hf_bucket/action.yml index dc72a03524f8..d0269ee22410 100644 --- a/.github/services/hf/hf_bucket/action.yml +++ b/.github/services/hf/hf_bucket/action.yml @@ -27,7 +27,12 @@ runs: export-env: true env: OPENDAL_HF_TOKEN: op://services/hf/token - OPENDAL_HF_REPO_ID: op://services/hf/bucket_repo_id + - name: Create temp bucket + uses: ./.github/actions/hf-temp-repo + with: + repo_id: opendal/test-bucket-${{ github.run_id }}-${{ github.job }} + repo_type: bucket + token: ${{ env.OPENDAL_HF_TOKEN }} - name: Setup env shell: bash run: | diff --git a/.github/services/hf/hf_dataset/action.yml b/.github/services/hf/hf_dataset/action.yml index 1659bc7305f4..caf2dc63733b 100644 --- a/.github/services/hf/hf_dataset/action.yml +++ b/.github/services/hf/hf_dataset/action.yml @@ -27,7 +27,12 @@ runs: export-env: true env: OPENDAL_HF_TOKEN: op://services/hf/token - OPENDAL_HF_REPO_ID: op://services/hf/dataset_repo_id + - name: Create temp dataset repo + uses: ./.github/actions/hf-temp-repo + with: + repo_id: opendal/test-dataset-${{ github.run_id }}-${{ github.job }} + repo_type: dataset + token: ${{ env.OPENDAL_HF_TOKEN }} - name: Setup env shell: bash run: | diff --git a/core/services/hf/src/lib.rs b/core/services/hf/src/lib.rs index b50185d55d4d..d0e8cc8844a6 100644 --- a/core/services/hf/src/lib.rs +++ b/core/services/hf/src/lib.rs @@ -62,3 +62,4 @@ mod tests { assert_eq!(HUGGINGFACE_SCHEME, "huggingface"); } } +